| 1 | /* |
| 2 | +----------------------------------------------------------------------+ |
| 3 | | Zend JIT | |
| 4 | +----------------------------------------------------------------------+ |
| 5 | | Copyright (c) The PHP Group | |
| 6 | +----------------------------------------------------------------------+ |
| 7 | | This source file is subject to version 3.01 of the PHP license, | |
| 8 | | that is bundled with this package in the file LICENSE, and is | |
| 9 | | available through the world-wide-web at the following url: | |
| 10 | | https://www.php.net/license/3_01.txt | |
| 11 | | If you did not receive a copy of the PHP license and are unable to | |
| 12 | | obtain it through the world-wide-web, please send a note to | |
| 13 | | license@php.net so we can mail you a copy immediately. | |
| 14 | +----------------------------------------------------------------------+ |
| 15 | | Authors: Dmitry Stogov <dmitry@php.net> | |
| 16 | +----------------------------------------------------------------------+ |
| 17 | */ |
| 18 | |
| 19 | static zend_op_array dummy_op_array; |
| 20 | static zend_jit_trace_info *zend_jit_traces = NULL; |
| 21 | static const void **zend_jit_exit_groups = NULL; |
| 22 | |
| 23 | #define ZEND_JIT_COUNTER_NUM zend_jit_traces[0].root |
| 24 | #define ZEND_JIT_TRACE_NUM zend_jit_traces[0].id |
| 25 | #define ZEND_JIT_EXIT_NUM zend_jit_traces[0].exit_count |
| 26 | #define ZEND_JIT_EXIT_COUNTERS zend_jit_traces[0].exit_counters |
| 27 | |
| 28 | #define ZEND_JIT_TRACE_STOP_DESCRIPTION(name, description) \ |
| 29 |  description, |
| 30 | |
| 31 | static const char * zend_jit_trace_stop_description[] = { |
| 32 |  ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_DESCRIPTION) |
| 33 | }; |
| 34 | |
| 35 | static zend_always_inline const char *zend_jit_trace_star_desc(uint8_t trace_flags) |
| 36 | { |
| 37 |  if (trace_flags & ZEND_JIT_TRACE_START_LOOP) { |
| 38 |    return "loop"; |
| 39 |  } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) { |
| 40 |    return "enter"; |
| 41 |  } else if (trace_flags & ZEND_JIT_TRACE_START_RETURN) { |
| 42 |    return "return"; |
| 43 |  } else { |
| 44 |    ZEND_UNREACHABLE(); |
| 45 |    return "???"; |
| 46 |  } |
| 47 | } |
| 48 | |
| 49 | static int zend_jit_trace_startup(zend_bool reattached) |
| 50 | { |
| 51 |  if (!reattached) { |
| 52 |    zend_jit_traces = (zend_jit_trace_info*)zend_shared_alloc(sizeof(zend_jit_trace_info) * JIT_G(max_root_traces)); |
| 53 |    if (!zend_jit_traces) { |
| 54 |      return FAILURE; |
| 55 |    } |
| 56 |    zend_jit_exit_groups = (const void**)zend_shared_alloc(sizeof(void*) * (ZEND_JIT_TRACE_MAX_EXITS/ZEND_JIT_EXIT_POINTS_PER_GROUP)); |
| 57 |    if (!zend_jit_exit_groups) { |
| 58 |      return FAILURE; |
| 59 |    } |
| 60 |    ZEND_JIT_TRACE_NUM = 1; |
| 61 |    ZEND_JIT_COUNTER_NUM = 0; |
| 62 |    ZEND_JIT_EXIT_NUM = 0; |
| 63 |    ZEND_JIT_EXIT_COUNTERS = 0; |
| 64 |    ZCSG(jit_traces) = zend_jit_traces; |
| 65 |    ZCSG(jit_exit_groups) = zend_jit_exit_groups; |
| 66 |  } else { |
| 67 |    zend_jit_traces = ZCSG(jit_traces); |
| 68 |    if (!zend_jit_traces) { |
| 69 |      return FAILURE; |
| 70 |    } |
| 71 |    zend_jit_exit_groups = ZCSG(jit_exit_groups); |
| 72 |    if (!zend_jit_exit_groups) { |
| 73 |      return FAILURE; |
| 74 |    } |
| 75 |  } |
| 76 | |
| 77 |  memset(&dummy_op_array, 0, sizeof(dummy_op_array)); |
| 78 |  dummy_op_array.fn_flags = ZEND_ACC_DONE_PASS_TWO; |
| 79 | |
| 80 |  JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1); |
| 81 |  if (JIT_G(exit_counters) == NULL) { |
| 82 |    return FAILURE; |
| 83 |  } |
| 84 | |
| 85 |  return SUCCESS; |
| 86 | } |
| 87 | |
| 88 | static const void *zend_jit_trace_allocate_exit_group(uint32_t n) |
| 89 | { |
| 90 |  dasm_State* dasm_state = NULL; |
| 91 |  const void *entry; |
| 92 |  char name[32]; |
| 93 | |
| 94 |  dasm_init(&dasm_state, DASM_MAXSECTION); |
| 95 |  dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); |
| 96 |  dasm_setup(&dasm_state, dasm_actions); |
| 97 |  zend_jit_trace_exit_group_stub(&dasm_state, n); |
| 98 | |
| 99 |  sprintf(name, "jit$$trace_exit_%d", n); |
| 100 |  entry = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, 0, SP_ADJ_JIT, SP_ADJ_NONE); |
| 101 |  dasm_free(&dasm_state); |
| 102 | |
| 103 | #ifdef HAVE_DISASM |
| 104 |  if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) { |
| 105 |    uint32_t i; |
| 106 | |
| 107 |    for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { |
| 108 |      sprintf(name, "jit$$trace_exit_%d", n + i); |
| 109 |      zend_jit_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING); |
| 110 |    } |
| 111 |  } |
| 112 | #endif |
| 113 | |
| 114 |  return entry; |
| 115 | } |
| 116 | |
| 117 | static const void *zend_jit_trace_allocate_exit_point(uint32_t n) |
| 118 | { |
| 119 |  const void *group = NULL; |
| 120 | |
| 121 |  if (UNEXPECTED(n >= ZEND_JIT_TRACE_MAX_EXITS)) { |
| 122 |    return NULL; |
| 123 |  } |
| 124 |  do { |
| 125 |    group = zend_jit_trace_allocate_exit_group(ZEND_JIT_EXIT_NUM); |
| 126 |    if (!group) { |
| 127 |      return NULL; |
| 128 |    } |
| 129 |    zend_jit_exit_groups[ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP] = |
| 130 |      group; |
| 131 |    ZEND_JIT_EXIT_NUM += ZEND_JIT_EXIT_POINTS_PER_GROUP; |
| 132 |  } while (n >= ZEND_JIT_EXIT_NUM); |
| 133 |  return (const void*) |
| 134 |    ((const char*)group + |
| 135 |    ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING)); |
| 136 | } |
| 137 | |
| 138 | static const void *zend_jit_trace_get_exit_addr(uint32_t n) |
| 139 | { |
| 140 |  if (UNEXPECTED(n >= ZEND_JIT_EXIT_NUM)) { |
| 141 |    return zend_jit_trace_allocate_exit_point(n); |
| 142 |  } |
| 143 |  return (const void*) |
| 144 |    ((const char*)zend_jit_exit_groups[n / ZEND_JIT_EXIT_POINTS_PER_GROUP] + |
| 145 |    ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING)); |
| 146 | } |
| 147 | |
| 148 | #if ZEND_JIT_TARGET_ARM64 |
| 149 | static zend_jit_trace_info *zend_jit_get_current_trace_info(void) |
| 150 | { |
| 151 |  return &zend_jit_traces[ZEND_JIT_TRACE_NUM]; |
| 152 | } |
| 153 | |
| 154 | static uint32_t zend_jit_trace_find_exit_point(const void* addr) |
| 155 | { |
| 156 |  uint32_t n = ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP; |
| 157 |  uint32_t i; |
| 158 | |
| 159 |  for (i = 0; i < n; i++) { |
| 160 |    if ((const char*)addr >= (const char*)zend_jit_exit_groups[i] |
| 161 |      && (const char*)addr < (const char*)zend_jit_exit_groups[i] + |
| 162 |        (ZEND_JIT_EXIT_POINTS_PER_GROUP * ZEND_JIT_EXIT_POINTS_SPACING)) { |
| 163 |      return (i * ZEND_JIT_EXIT_POINTS_PER_GROUP) + |
| 164 |         ((const char*)addr - (const char*)zend_jit_exit_groups[i]) / ZEND_JIT_EXIT_POINTS_SPACING; |
| 165 |    } |
| 166 |  } |
| 167 |  return (uint32_t)-1; |
| 168 | } |
| 169 | #endif |
| 170 | |
| 171 | static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags) |
| 172 | { |
| 173 |  zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; |
| 174 |  uint32_t exit_point; |
| 175 |  const zend_op_array *op_array; |
| 176 |  uint32_t stack_offset = (uint32_t)-1; |
| 177 |  uint32_t stack_size; |
| 178 |  zend_jit_trace_stack *stack = NULL; |
| 179 | |
| 180 |  if (delayed_call_chain) { |
| 181 |    assert(to_opline != NULL); /* CALL and IP share the same register */ |
| 182 |    flags |= ZEND_JIT_EXIT_RESTORE_CALL; |
| 183 |  } |
| 184 |  if (JIT_G(current_frame)) { |
| 185 |    op_array = &JIT_G(current_frame)->func->op_array; |
| 186 |    stack_size = op_array->last_var + op_array->T; |
| 187 |    if (stack_size) { |
| 188 |      stack = JIT_G(current_frame)->stack; |
| 189 |      do { |
| 190 |        if (STACK_TYPE(stack, stack_size-1) != IS_UNKNOWN |
| 191 |         || STACK_MEM_TYPE(stack, stack_size-1) != IS_UNKNOWN |
| 192 |         || STACK_REG(stack, stack_size-1) != ZREG_NONE) { |
| 193 |          break; |
| 194 |        } |
| 195 |        stack_size--; |
| 196 |      } while (stack_size); |
| 197 |    } |
| 198 |  } else { |
| 199 |    op_array = NULL; |
| 200 |    stack_size = 0; |
| 201 |  } |
| 202 | |
| 203 |  /* Try to reuse exit points */ |
| 204 |  if (to_opline != NULL && t->exit_count > 0) { |
| 205 |    uint32_t i = t->exit_count; |
| 206 | |
| 207 |    do { |
| 208 |      i--; |
| 209 |      if (stack_size == 0 |
| 210 |       || (t->exit_info[i].stack_size >= stack_size |
| 211 |       && memcmp(t->stack_map + t->exit_info[i].stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)) == 0)) { |
| 212 |        stack_offset = t->exit_info[i].stack_offset; |
| 213 |        if (t->exit_info[i].opline == to_opline |
| 214 |         && t->exit_info[i].flags == flags |
| 215 |         && t->exit_info[i].stack_size == stack_size) { |
| 216 |          return i; |
| 217 |        } |
| 218 |      } |
| 219 |    } while (i > 0); |
| 220 |  } |
| 221 | |
| 222 |  exit_point = t->exit_count; |
| 223 |  if (exit_point < ZEND_JIT_TRACE_MAX_EXITS) { |
| 224 |    if (stack_size != 0 && stack_offset == (uint32_t)-1) { |
| 225 |      stack_offset = t->stack_map_size; |
| 226 |      t->stack_map_size += stack_size; |
| 227 |      // TODO: reduce number of reallocations ??? |
| 228 |      t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack)); |
| 229 |      memcpy(t->stack_map + stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)); |
| 230 |    } |
| 231 |    t->exit_count++; |
| 232 |    t->exit_info[exit_point].opline = to_opline; |
| 233 |    t->exit_info[exit_point].op_array = op_array; |
| 234 |    t->exit_info[exit_point].flags = flags; |
| 235 |    t->exit_info[exit_point].stack_size = stack_size; |
| 236 |    t->exit_info[exit_point].stack_offset = stack_offset; |
| 237 |  } |
| 238 | |
| 239 |  return exit_point; |
| 240 | } |
| 241 | |
| 242 | static void zend_jit_trace_add_code(const void *start, uint32_t size) |
| 243 | { |
| 244 |  zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; |
| 245 | |
| 246 |  t->code_start = start; |
| 247 |  t->code_size = size; |
| 248 | } |
| 249 | |
| 250 | static uint32_t zend_jit_find_trace(const void *addr) |
| 251 | { |
| 252 |  uint32_t i; |
| 253 | |
| 254 |  for (i = 1; i < ZEND_JIT_TRACE_NUM; i++) { |
| 255 |    if (zend_jit_traces[i].code_start == addr) { |
| 256 |      return i; |
| 257 |    } |
| 258 |  } |
| 259 |  ZEND_UNREACHABLE(); |
| 260 |  return 0; |
| 261 | } |
| 262 | |
| 263 | static zend_string *zend_jit_trace_name(const zend_op_array *op_array, uint32_t lineno) |
| 264 | { |
| 265 |  smart_str buf = {0}; |
| 266 | |
| 267 |  smart_str_appends(&buf, TRACE_PREFIX); |
| 268 |  smart_str_append_long(&buf, (zend_long)ZEND_JIT_TRACE_NUM); |
| 269 |  smart_str_appendc(&buf, '$'); |
| 270 |  if (op_array->function_name) { |
| 271 |    if (op_array->scope) { |
| 272 |      smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name)); |
| 273 |      smart_str_appends(&buf, "::"); |
| 274 |      smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name)); |
| 275 |    } else { |
| 276 |      smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name)); |
| 277 |    } |
| 278 |  } else if (op_array->filename) { |
| 279 |    smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename)); |
| 280 |  } |
| 281 |  smart_str_appendc(&buf, '$'); |
| 282 |  smart_str_append_long(&buf, (zend_long)lineno); |
| 283 |  smart_str_0(&buf); |
| 284 |  return buf.s; |
| 285 | } |
| 286 | |
| 287 | static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline) |
| 288 | { |
| 289 |  switch (opline->opcode) { |
| 290 |    case ZEND_IS_IDENTICAL: |
| 291 |    case ZEND_IS_NOT_IDENTICAL: |
| 292 |    case ZEND_IS_EQUAL: |
| 293 |    case ZEND_IS_NOT_EQUAL: |
| 294 |    case ZEND_IS_SMALLER: |
| 295 |    case ZEND_IS_SMALLER_OR_EQUAL: |
| 296 |    case ZEND_CASE: |
| 297 |    case ZEND_CASE_STRICT: |
| 298 |    case ZEND_ISSET_ISEMPTY_CV: |
| 299 |    case ZEND_ISSET_ISEMPTY_VAR: |
| 300 |    case ZEND_ISSET_ISEMPTY_DIM_OBJ: |
| 301 |    case ZEND_ISSET_ISEMPTY_PROP_OBJ: |
| 302 |    case ZEND_ISSET_ISEMPTY_STATIC_PROP: |
| 303 |    case ZEND_INSTANCEOF: |
| 304 |    case ZEND_TYPE_CHECK: |
| 305 |    case ZEND_DEFINED: |
| 306 |    case ZEND_IN_ARRAY: |
| 307 |    case ZEND_ARRAY_KEY_EXISTS: |
| 308 |      if (opline->result_type & (IS_SMART_BRANCH_JMPNZ | IS_SMART_BRANCH_JMPZ)) { |
| 309 |        /* smart branch */ |
| 310 |        return 1; |
| 311 |      } |
| 312 |      break; |
| 313 |    case ZEND_JMPZ: |
| 314 |    case ZEND_JMPNZ: |
| 315 |    case ZEND_JMPZ_EX: |
| 316 |    case ZEND_JMPNZ_EX: |
| 317 |    case ZEND_JMP_SET: |
| 318 |    case ZEND_COALESCE: |
| 319 |    case ZEND_JMP_NULL: |
| 320 |    case ZEND_FE_RESET_R: |
| 321 |    case ZEND_FE_RESET_RW: |
| 322 |    case ZEND_ASSERT_CHECK: |
| 323 |    case ZEND_FE_FETCH_R: |
| 324 |    case ZEND_FE_FETCH_RW: |
| 325 |    case ZEND_SWITCH_LONG: |
| 326 |    case ZEND_SWITCH_STRING: |
| 327 |    case ZEND_MATCH: |
| 328 |      /* branch opcodes */ |
| 329 |      return 1; |
| 330 |    case ZEND_NEW: |
| 331 |      if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) { |
| 332 |        /* NEW may skip constructor without arguments */ |
| 333 |        return 1; |
| 334 |      } |
| 335 |      break; |
| 336 |    case ZEND_CATCH: |
| 337 |    case ZEND_FAST_CALL: |
| 338 |    case ZEND_FAST_RET: |
| 339 |    case ZEND_GENERATOR_CREATE: |
| 340 |    case ZEND_GENERATOR_RETURN: |
| 341 |    case ZEND_EXIT: |
| 342 |    case ZEND_YIELD: |
| 343 |    case ZEND_YIELD_FROM: |
| 344 |    case ZEND_INCLUDE_OR_EVAL: |
| 345 |    case ZEND_MATCH_ERROR: |
| 346 |      /* unsupported */ |
| 347 |      return 1; |
| 348 |    case ZEND_DO_FCALL: |
| 349 |      /* potentially polymorphic call */ |
| 350 |      return 1; |
| 351 | #if 0 |
| 352 |    case ZEND_DO_UCALL: |
| 353 |    case ZEND_DO_FCALL_BY_NAME: |
| 354 |      /* monomorphic call */ |
| 355 |      // TODO: recompilation may change target ??? |
| 356 |      return 0; |
| 357 | #endif |
| 358 |    case ZEND_RETURN_BY_REF: |
| 359 |    case ZEND_RETURN: |
| 360 |      /* return */ |
| 361 |      return !JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)); |
| 362 |    default: |
| 363 |      break; |
| 364 |  } |
| 365 |  return 0; |
| 366 | } |
| 367 | |
| 368 | static zend_always_inline uint32_t zend_jit_trace_type_to_info_ex(zend_uchar type, uint32_t info) |
| 369 | { |
| 370 |  if (type == IS_UNKNOWN) { |
| 371 |    return info; |
| 372 |  } |
| 373 |  ZEND_ASSERT(info & (1 << type)); |
| 374 |  if (type < IS_STRING) { |
| 375 |    return (1 << type); |
| 376 |  } else if (type != IS_ARRAY) { |
| 377 |    return (1 << type) | (info & (MAY_BE_RC1|MAY_BE_RCN)); |
| 378 |  } else { |
| 379 |    return MAY_BE_ARRAY | (info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); |
| 380 |  } |
| 381 | } |
| 382 | |
| 383 | static zend_always_inline uint32_t zend_jit_trace_type_to_info(zend_uchar type) |
| 384 | { |
| 385 |  return zend_jit_trace_type_to_info_ex(type, -1); |
| 386 | } |
| 387 | |
| 388 | static zend_always_inline zend_ssa_alias_kind zend_jit_var_may_alias(const zend_op_array *op_array, const zend_ssa *ssa, uint32_t var) |
| 389 | { |
| 390 |  if (var >= op_array->last_var) { |
| 391 |    return NO_ALIAS; |
| 392 |  } else if (( )) { |
| 393 |    return SYMTABLE_ALIAS; |
| 394 |  } else if (ssa->vars) { |
| 395 |    return ssa->vars[var].alias; |
| 396 |  } else if (zend_string_equals_literal(op_array->vars[var], "http_response_header")) { |
| 397 |    return HTTP_RESPONSE_HEADER_ALIAS; |
| 398 |  } |
| 399 |  return NO_ALIAS; |
| 400 | } |
| 401 | |
| 402 | static zend_always_inline void zend_jit_trace_add_op_guard(zend_ssa *tssa, |
| 403 | int ssa_var, |
| 404 | uint8_t op_type) |
| 405 | { |
| 406 |  zend_ssa_var_info *info = &tssa->var_info[ssa_var]; |
| 407 | |
| 408 |  if ( ) { |
| 409 |    if (UNEXPECTED(tssa->vars[ssa_var].alias != NO_ALIAS)) { |
| 410 |      info->type |= MAY_BE_GUARD; |
| 411 |    } else { |
| 412 |      info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op_type, info->type); |
| 413 |    } |
| 414 |  } |
| 415 | } |
| 416 | |
| 417 | #define ADD_OP_GUARD(_ssa_var, _op_type) do { \ |
| 418 |    if (_ssa_var >= 0 && _op_type != IS_UNKNOWN) { \ |
| 419 |      zend_jit_trace_add_op_guard(tssa, _ssa_var, _op_type); \ |
| 420 |    } \ |
| 421 |  } while (0) |
| 422 | |
| 423 | #define CHECK_OP_TRACE_TYPE(_var, _ssa_var, op_info, op_type) do { \ |
| 424 |    if (op_type != IS_UNKNOWN) { \ |
| 425 |      if ((op_info & MAY_BE_GUARD) != 0) { \ |
| 426 |        if (!zend_jit_type_guard(&dasm_state, opline, _var, op_type)) { \ |
| 427 |          goto jit_failure; \ |
| 428 |        } \ |
| 429 |        if (ssa->vars[_ssa_var].alias != NO_ALIAS) { \ |
| 430 |          SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), IS_UNKNOWN, 1); \ |
| 431 |          op_info = zend_jit_trace_type_to_info(op_type); \ |
| 432 |        } else { \ |
| 433 |          SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), op_type, 1); \ |
| 434 |          op_info &= ~MAY_BE_GUARD; \ |
| 435 |          ssa->var_info[_ssa_var].type &= op_info; \ |
| 436 |        } \ |
| 437 |      } \ |
| 438 |    } \ |
| 439 |  } while (0) |
| 440 | |
| 441 | #define ADD_OP1_TRACE_GUARD() \ |
| 442 |  ADD_OP_GUARD(tssa->ops[idx].op1_use, op1_type) |
| 443 | #define ADD_OP2_TRACE_GUARD() \ |
| 444 |  ADD_OP_GUARD(tssa->ops[idx].op2_use, op2_type) |
| 445 | #define ADD_OP1_DATA_TRACE_GUARD() \ |
| 446 |  ADD_OP_GUARD(tssa->ops[idx+1].op1_use, op3_type) |
| 447 | |
| 448 | #define CHECK_OP1_TRACE_TYPE() \ |
| 449 |  CHECK_OP_TRACE_TYPE(opline->op1.var, ssa_op->op1_use, op1_info, op1_type) |
| 450 | #define CHECK_OP2_TRACE_TYPE() \ |
| 451 |  CHECK_OP_TRACE_TYPE(opline->op2.var, ssa_op->op2_use, op2_info, op2_type) |
| 452 | #define CHECK_OP1_DATA_TRACE_TYPE() \ |
| 453 |  CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type) |
| 454 | |
| 455 | static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array) |
| 456 | { |
| 457 |  if (op_array && op_array->type == ZEND_USER_FUNCTION) { |
| 458 |    return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack))); |
| 459 |  } else if (op_array) { |
| 460 |    return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(op_array->num_args * sizeof(zend_jit_trace_stack))); |
| 461 |  } else { |
| 462 |    return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack)); |
| 463 |  } |
| 464 | } |
| 465 | |
| 466 | static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array) |
| 467 | { |
| 468 |  return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array)); |
| 469 | } |
| 470 | |
| 471 | static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array) |
| 472 | { |
| 473 |  return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array)); |
| 474 | } |
| 475 | |
| 476 | static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, zend_uchar type) |
| 477 | { |
| 478 |  zend_jit_trace_stack *stack = call->stack; |
| 479 |  const zend_op_array *op_array = &call->func->op_array; |
| 480 |  uint32_t arg_num = opline->op2.num; |
| 481 | |
| 482 |  if (arg_num > op_array->num_args) { |
| 483 |    return; |
| 484 |  } |
| 485 |  if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { |
| 486 |    zend_arg_info *arg_info; |
| 487 | |
| 488 |    ZEND_ASSERT(arg_num <= op_array->num_args); |
| 489 |    arg_info = &op_array->arg_info[arg_num-1]; |
| 490 | |
| 491 |    if (ZEND_TYPE_IS_SET(arg_info->type)) { |
| 492 |      if (!(ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) { |
| 493 |        return; |
| 494 |      } |
| 495 |    } |
| 496 |  } |
| 497 |  SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1); |
| 498 | } |
| 499 | |
| 500 | static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info) |
| 501 | { |
| 502 |  if (func |
| 503 |   && func->type == ZEND_INTERNAL_FUNCTION |
| 504 |   && (func->internal_function.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0 |
| 505 |   && arg_num < func->internal_function.num_args) { |
| 506 |    const zend_internal_arg_info *arg_info = &func->internal_function.arg_info[arg_num]; |
| 507 | |
| 508 |    if (ZEND_ARG_SEND_MODE(arg_info) == ZEND_SEND_BY_VAL |
| 509 |     && ZEND_TYPE_IS_SET(arg_info->type) |
| 510 |     && (ZEND_TYPE_FULL_MASK(arg_info->type) & MAY_BE_ANY) != MAY_BE_ANY) { |
| 511 |      if ( |
| 512 |       |
| 513 |       |
| 514 |       ) { |
| 515 |        uint32_t type = STACK_TYPE(JIT_G(current_frame)->call->stack, arg_num); |
| 516 | |
| 517 |        if (type != IS_UNKNOWN |
| 518 |         && type < IS_STRING |
| 519 |         && ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type)) { |
| 520 |          return 0; |
| 521 |        } |
| 522 |      } |
| 523 |      if (call_info && arg_num < call_info->num_args && call_info->arg_info[arg_num].opline) { |
| 524 |        const zend_op *opline = call_info->arg_info[arg_num].opline; |
| 525 | |
| 526 |        if (opline->opcode == ZEND_SEND_VAL && opline->op1_type == IS_CONST) { |
| 527 |          zval *zv = RT_CONSTANT(opline, opline->op1); |
| 528 | |
| 529 |          if (!Z_REFCOUNTED_P(zv)) { |
| 530 |            uint32_t type = Z_TYPE_P(zv); |
| 531 | |
| 532 |            // TODO: few functions (e.g. pcntl_exec) modify arrays in-place ??? |
| 533 |            if (type != IS_ARRAY |
| 534 |             && (ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) { |
| 535 |              return 0; |
| 536 |            } |
| 537 |          } |
| 538 |        } |
| 539 |      } |
| 540 |    } |
| 541 |  } |
| 542 | |
| 543 |  return 1; |
| 544 | } |
| 545 | |
| 546 | static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_script *script) |
| 547 | { |
| 548 |  zend_jit_op_array_trace_extension *jit_extension; |
| 549 |  zend_ssa *ssa; |
| 550 | |
| 551 |  jit_extension = |
| 552 |    (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 553 |  jit_extension->func_info.num = 0; |
| 554 |  jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC |
| 555 |    | ZEND_FUNC_JIT_ON_PROF_REQUEST |
| 556 |    | ZEND_FUNC_JIT_ON_HOT_COUNTERS |
| 557 |    | ZEND_FUNC_JIT_ON_HOT_TRACE; |
| 558 |  memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa)); |
| 559 |  ssa = &jit_extension->func_info.ssa; |
| 560 | |
| 561 |  if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) { |
| 562 |    do { |
| 563 |      if (zend_jit_op_array_analyze1(op_array, script, ssa) != SUCCESS) { |
| 564 |        break; |
| 565 |      } |
| 566 | |
| 567 |      if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) { |
| 568 |        zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, (zend_op_array*)op_array, &jit_extension->func_info); |
| 569 |        jit_extension->func_info.call_map = zend_build_call_map(&CG(arena), &jit_extension->func_info, op_array); |
| 570 |        if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) { |
| 571 |          zend_init_func_return_info(op_array, script, &jit_extension->func_info.return_info); |
| 572 |        } |
| 573 |      } |
| 574 | |
| 575 |      if (zend_jit_op_array_analyze2(op_array, script, ssa, 0) != SUCCESS) { |
| 576 |        break; |
| 577 |      } |
| 578 | |
| 579 |      if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) { |
| 580 |        zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", ssa); |
| 581 |      } |
| 582 |      return ssa; |
| 583 |    } while (0); |
| 584 |  } |
| 585 | |
| 586 |  memset(ssa, 0, sizeof(zend_ssa)); |
| 587 |  ssa->cfg.blocks_count = 1; |
| 588 | |
| 589 |  if (JIT_G(opt_level) == ZEND_JIT_LEVEL_INLINE) { |
| 590 |    zend_cfg cfg; |
| 591 |    void *checkpoint = zend_arena_checkpoint(CG(arena)); |
| 592 | |
| 593 |    if (zend_jit_build_cfg(op_array, &cfg) == SUCCESS) { |
| 594 |      ssa->cfg.flags = cfg.flags; |
| 595 |    } else{ |
| 596 |      ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS; |
| 597 |    } |
| 598 | |
| 599 |    /* TODO: move this to zend_cfg.c ? */ |
| 600 |    if (!op_array->function_name) { |
| 601 |      ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS; |
| 602 |    } |
| 603 | |
| 604 |    zend_arena_release(&CG(arena), checkpoint); |
| 605 |  } |
| 606 | |
| 607 |  return ssa; |
| 608 | } |
| 609 | |
| 610 | static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa); |
| 611 | static void zend_jit_dump_exit_info(zend_jit_trace_info *t); |
| 612 | |
| 613 | static zend_always_inline int zend_jit_trace_op_len(const zend_op *opline) |
| 614 | { |
| 615 |  int len; |
| 616 | |
| 617 |  switch (opline->opcode) { |
| 618 |    case ZEND_ASSIGN_DIM: |
| 619 |    case ZEND_ASSIGN_OBJ: |
| 620 |    case ZEND_ASSIGN_STATIC_PROP: |
| 621 |    case ZEND_ASSIGN_DIM_OP: |
| 622 |    case ZEND_ASSIGN_OBJ_OP: |
| 623 |    case ZEND_ASSIGN_STATIC_PROP_OP: |
| 624 |    case ZEND_ASSIGN_OBJ_REF: |
| 625 |    case ZEND_ASSIGN_STATIC_PROP_REF: |
| 626 |      return 2; /* OP_DATA */ |
| 627 |    case ZEND_RECV_INIT: |
| 628 |      len = 1; |
| 629 |      opline++; |
| 630 |      while (opline->opcode == ZEND_RECV_INIT) { |
| 631 |        len++; |
| 632 |        opline++; |
| 633 |      } |
| 634 |      return len; |
| 635 |    case ZEND_BIND_GLOBAL: |
| 636 |      len = 1; |
| 637 |      opline++; |
| 638 |      while (opline->opcode == ZEND_BIND_GLOBAL) { |
| 639 |        len++; |
| 640 |        opline++; |
| 641 |      } |
| 642 |      return len; |
| 643 | //    case ZEND_IS_IDENTICAL: |
| 644 | //    case ZEND_IS_NOT_IDENTICAL: |
| 645 | //    case ZEND_IS_EQUAL: |
| 646 | //    case ZEND_IS_NOT_EQUAL: |
| 647 | //    case ZEND_IS_SMALLER: |
| 648 | //    case ZEND_IS_SMALLER_OR_EQUAL: |
| 649 | //    case ZEND_CASE: |
| 650 | //    case ZEND_ISSET_ISEMPTY_CV: |
| 651 | //    case ZEND_ISSET_ISEMPTY_VAR: |
| 652 | //    case ZEND_ISSET_ISEMPTY_DIM_OBJ: |
| 653 | //    case ZEND_ISSET_ISEMPTY_PROP_OBJ: |
| 654 | //    case ZEND_ISSET_ISEMPTY_STATIC_PROP: |
| 655 | //    case ZEND_INSTANCEOF: |
| 656 | //    case ZEND_TYPE_CHECK: |
| 657 | //    case ZEND_DEFINED: |
| 658 | //    case ZEND_IN_ARRAY: |
| 659 | //    case ZEND_ARRAY_KEY_EXISTS: |
| 660 |    default: |
| 661 |      if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { |
| 662 |        return 2; /* JMPZ/JMPNZ */ |
| 663 |      } |
| 664 |      return 1; |
| 665 |  } |
| 666 | } |
| 667 | |
| 668 | static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack) |
| 669 | { |
| 670 |  const zend_op_array *op_array; |
| 671 |  zend_jit_trace_rec *p; |
| 672 |  int k, vars_count; |
| 673 |  zend_bitset use, def; |
| 674 |  uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS; |
| 675 |  uint32_t set_size; |
| 676 |  zend_ssa_phi *prev = NULL; |
| 677 |  int level = 0; |
| 678 |  ALLOCA_FLAG(use_heap); |
| 679 | |
| 680 |  op_array = trace_buffer->op_array; |
| 681 |  set_size = zend_bitset_len(op_array->last_var + op_array->T); |
| 682 |  use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap); |
| 683 |  memset(use, 0, set_size * 2 * ZEND_BITSET_ELM_SIZE); |
| 684 |  def = use + set_size; |
| 685 |  p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; |
| 686 |  for (;;p++) { |
| 687 |    if (p->op == ZEND_JIT_TRACE_VM && level == 0) { |
| 688 |      const zend_op *opline = p->opline; |
| 689 |      int len; |
| 690 | |
| 691 |      zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def); |
| 692 |      len = zend_jit_trace_op_len(opline); |
| 693 |      while (len > 1) { |
| 694 |        opline++; |
| 695 |        if (opline->opcode != ZEND_OP_DATA) { |
| 696 |          zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def); |
| 697 |        } |
| 698 |        len--; |
| 699 |      } |
| 700 |    } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { |
| 701 |    } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { |
| 702 |    } else if (p->op == ZEND_JIT_TRACE_ENTER) { |
| 703 |      level++; |
| 704 |    } else if (p->op == ZEND_JIT_TRACE_BACK) { |
| 705 |      if (level == 0) { |
| 706 |        // Phi for recursive calls and returns are not supported yet ??? |
| 707 |        assert(0); |
| 708 |      } else { |
| 709 |        level--; |
| 710 |      } |
| 711 |    } else if (p->op == ZEND_JIT_TRACE_END) { |
| 712 |      break; |
| 713 |    } |
| 714 |  } |
| 715 | |
| 716 |  zend_bitset_intersection(use, def, set_size); |
| 717 | |
| 718 |  if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { |
| 719 |    vars_count = op_array->last_var; |
| 720 |  } else { |
| 721 |    vars_count = op_array->last_var + op_array->T; |
| 722 |  } |
| 723 |  for (k = 0; k < vars_count; k++) { |
| 724 |    if (zend_bitset_in(use, k)) { |
| 725 |      zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1, |
| 726 |        ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) + |
| 727 |        ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) + |
| 728 |        sizeof(void*) * 2); |
| 729 |      phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi))); |
| 730 |      phi->sources[0] = STACK_VAR(stack, k); |
| 731 |      phi->sources[1] = -1; |
| 732 |      phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2)); |
| 733 |      phi->pi = -1; |
| 734 |      phi->var = k; |
| 735 |      phi->ssa_var = ssa_vars_count; |
| 736 |      SET_STACK_VAR(stack, k, ssa_vars_count); |
| 737 |      ssa_vars_count++; |
| 738 |      phi->block = 1; |
| 739 |      if (prev) { |
| 740 |        prev->next = phi; |
| 741 |      } else { |
| 742 |        tssa->blocks[1].phis = phi; |
| 743 |      } |
| 744 |      prev = phi; |
| 745 |    } |
| 746 |  } |
| 747 | |
| 748 |  free_alloca(use, use_heap); |
| 749 | |
| 750 |  return ssa_vars_count; |
| 751 | } |
| 752 | |
| 753 | static int zend_jit_trace_add_call_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack) |
| 754 | { |
| 755 |  zend_ssa_phi *prev = NULL; |
| 756 |  const zend_op_array *op_array = trace_buffer->op_array; |
| 757 |  const zend_op *opline = trace_buffer[1].opline; |
| 758 |  int count = opline - op_array->opcodes; |
| 759 |  int i; |
| 760 | |
| 761 |  for(i = 0; i < count; i++) { |
| 762 |    zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1, |
| 763 |      ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) + |
| 764 |      ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) + |
| 765 |      sizeof(void*) * 2); |
| 766 |    phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi))); |
| 767 |    phi->sources[0] = STACK_VAR(stack, i); |
| 768 |    phi->sources[1] = -1; |
| 769 |    phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2)); |
| 770 |    phi->pi = -1; |
| 771 |    phi->var = i; |
| 772 |    phi->ssa_var = ssa_vars_count; |
| 773 |    SET_STACK_VAR(stack, i, ssa_vars_count); |
| 774 |    ssa_vars_count++; |
| 775 |    phi->block = 1; |
| 776 |    if (prev) { |
| 777 |      prev->next = phi; |
| 778 |    } else { |
| 779 |      tssa->blocks[1].phis = phi; |
| 780 |    } |
| 781 |    prev = phi; |
| 782 |  } |
| 783 |  return ssa_vars_count; |
| 784 | } |
| 785 | |
| 786 | static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack) |
| 787 | { |
| 788 |  const zend_op *opline = trace_buffer[1].opline - 1; |
| 789 |  int i; |
| 790 | |
| 791 |  if (RETURN_VALUE_USED(opline)) { |
| 792 |    zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1, |
| 793 |      ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) + |
| 794 |      ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) + |
| 795 |      sizeof(void*) * 2); |
| 796 | |
| 797 |    i = EX_VAR_TO_NUM(opline->result.var); |
| 798 |    phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi))); |
| 799 |    phi->sources[0] = STACK_VAR(stack, i); |
| 800 |    phi->sources[1] = -1; |
| 801 |    phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2)); |
| 802 |    phi->pi = -1; |
| 803 |    phi->var = i; |
| 804 |    phi->ssa_var = ssa_vars_count; |
| 805 |    SET_STACK_VAR(stack, i, ssa_vars_count); |
| 806 |    ssa_vars_count++; |
| 807 |    phi->block = 1; |
| 808 |    tssa->blocks[1].phis = phi; |
| 809 |  } |
| 810 |  return ssa_vars_count; |
| 811 | } |
| 812 | |
| 813 | static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) |
| 814 | { |
| 815 |  int var, use; |
| 816 |  zend_ssa_op *op; |
| 817 |  zend_ssa_var_info *info; |
| 818 |  unsigned int no_val; |
| 819 |  zend_ssa_alias_kind alias; |
| 820 | |
| 821 |  if (tssa->vars[ssa_var].phi_use_chain) { |
| 822 |    // TODO: this may be incorrect ??? |
| 823 |    var = tssa->vars[ssa_var].phi_use_chain->ssa_var; |
| 824 |  } else { |
| 825 |    var = ssa_var; |
| 826 |  } |
| 827 |  use = tssa->vars[var].use_chain; |
| 828 |  if (use >= 0) { |
| 829 |    ZEND_ASSERT((tssa_opcodes[use] - op_array->opcodes) < op_array->last); |
| 830 |    op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes); |
| 831 |    if (tssa->ops[use].op1_use == var) { |
| 832 |      no_val = ssa->vars[op->op1_use].no_val; |
| 833 |      alias = ssa->vars[op->op1_use].alias; |
| 834 |      info = ssa->var_info + op->op1_use; |
| 835 |    } else if (tssa->ops[use].op2_use == var) { |
| 836 |      no_val = ssa->vars[op->op2_use].no_val; |
| 837 |      alias = ssa->vars[op->op2_use].alias; |
| 838 |      info = ssa->var_info + op->op2_use; |
| 839 |    } else if (tssa->ops[use].result_use == var) { |
| 840 |      no_val = ssa->vars[op->result_use].no_val; |
| 841 |      alias = ssa->vars[op->result_use].alias; |
| 842 |      info = ssa->var_info + op->result_use; |
| 843 |    } else { |
| 844 |      assert(0); |
| 845 |      return 0; |
| 846 |    } |
| 847 |    tssa->vars[ssa_var].no_val = no_val; |
| 848 |    tssa->vars[ssa_var].alias = alias; |
| 849 |    memcpy(&tssa->var_info[ssa_var], info, sizeof(zend_ssa_var_info)); |
| 850 |    return 1; |
| 851 |  } |
| 852 |  return 0; |
| 853 | } |
| 854 | |
| 855 | static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) |
| 856 | { |
| 857 |  zend_ssa_range tmp; |
| 858 |  int def = tssa->vars[ssa_var].definition; |
| 859 | |
| 860 |  if (tssa->vars[ssa_var].alias == NO_ALIAS |
| 861 |   && zend_inference_propagate_range(op_array, tssa, (zend_op*)tssa_opcodes[def], (zend_ssa_op*)&tssa->ops[def], ssa_var, &tmp)) { |
| 862 |    tssa->var_info[ssa_var].range.min = tmp.min; |
| 863 |    tssa->var_info[ssa_var].range.max = tmp.max; |
| 864 |    tssa->var_info[ssa_var].range.underflow = tmp.underflow; |
| 865 |    tssa->var_info[ssa_var].range.overflow = tmp.overflow; |
| 866 |    tssa->var_info[ssa_var].has_range = 1; |
| 867 |  } |
| 868 | } |
| 869 | |
| 870 | static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) |
| 871 | { |
| 872 |  int def; |
| 873 |  zend_ssa_op *op; |
| 874 |  zend_ssa_var_info *info; |
| 875 |  unsigned int no_val; |
| 876 |  zend_ssa_alias_kind alias; |
| 877 | |
| 878 |  def = tssa->vars[ssa_var].definition; |
| 879 |  if (def >= 0) { |
| 880 |    ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last); |
| 881 |    op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes); |
| 882 |    if (tssa->ops[def].op1_def == ssa_var) { |
| 883 |      no_val = ssa->vars[op->op1_def].no_val; |
| 884 |      alias = ssa->vars[op->op1_def].alias; |
| 885 |      info = ssa->var_info + op->op1_def; |
| 886 |    } else if (tssa->ops[def].op2_def == ssa_var) { |
| 887 |      no_val = ssa->vars[op->op2_def].no_val; |
| 888 |      alias = ssa->vars[op->op2_def].alias; |
| 889 |      info = ssa->var_info + op->op2_def; |
| 890 |    } else if (tssa->ops[def].result_def == ssa_var) { |
| 891 |      no_val = ssa->vars[op->result_def].no_val; |
| 892 |      alias = ssa->vars[op->result_def].alias; |
| 893 |      info = ssa->var_info + op->result_def; |
| 894 |    } else { |
| 895 |      assert(0); |
| 896 |      return; |
| 897 |    } |
| 898 | |
| 899 |    tssa->vars[ssa_var].no_val = no_val; |
| 900 |    tssa->vars[ssa_var].alias = alias; |
| 901 | |
| 902 |    if (!(info->type & MAY_BE_REF)) { |
| 903 |      zend_jit_trace_propagate_range(op_array, tssa_opcodes, tssa, ssa_var); |
| 904 |    } |
| 905 | |
| 906 |    if (info->has_range) { |
| 907 |      if (tssa->var_info[ssa_var].has_range) { |
| 908 |        tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min); |
| 909 |        tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max); |
| 910 |        tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow; |
| 911 |        tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow; |
| 912 |      } else { |
| 913 |        tssa->var_info[ssa_var].has_range = 1; |
| 914 |        tssa->var_info[ssa_var].range = info->range; |
| 915 |      } |
| 916 |    } |
| 917 |  } |
| 918 | } |
| 919 | |
| 920 | static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var) |
| 921 | { |
| 922 |  int def; |
| 923 |  zend_ssa_op *op; |
| 924 |  zend_ssa_var_info *info; |
| 925 | |
| 926 |  def = tssa->vars[ssa_var].definition; |
| 927 |  if (def >= 0) { |
| 928 |    ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last); |
| 929 |    op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes); |
| 930 |    if (tssa->ops[def].op1_def == ssa_var) { |
| 931 |      info = ssa->var_info + op->op1_def; |
| 932 |    } else if (tssa->ops[def].op2_def == ssa_var) { |
| 933 |      info = ssa->var_info + op->op2_def; |
| 934 |    } else if (tssa->ops[def].result_def == ssa_var) { |
| 935 |      info = ssa->var_info + op->result_def; |
| 936 |    } else { |
| 937 |      assert(0); |
| 938 |      return 0; |
| 939 |    } |
| 940 |    tssa->var_info[ssa_var].type &= info->type; |
| 941 |    if (info->ce) { |
| 942 |      if (tssa->var_info[ssa_var].ce) { |
| 943 |        if (tssa->var_info[ssa_var].ce != info->ce) { |
| 944 |          if (instanceof_function(tssa->var_info[ssa_var].ce, info->ce)) { |
| 945 |            /* everything fine */ |
| 946 |          } else if (instanceof_function(info->ce, tssa->var_info[ssa_var].ce)) { |
| 947 |            // TODO: TSSA may miss Pi() functions and corresponding instanceof() constraints ??? |
| 948 |          } else { |
| 949 |            // TODO: classes may implement the same interface ??? |
| 950 |            //ZEND_UNREACHABLE(); |
| 951 |          } |
| 952 |        } |
| 953 |        tssa->var_info[ssa_var].is_instanceof = |
| 954 |          tssa->var_info[ssa_var].is_instanceof && info->is_instanceof; |
| 955 |      } else { |
| 956 |        tssa->var_info[ssa_var].ce = info->ce; |
| 957 |        tssa->var_info[ssa_var].is_instanceof = info->is_instanceof; |
| 958 |      } |
| 959 |    } |
| 960 |    if (info->has_range) { |
| 961 |      if (tssa->var_info[ssa_var].has_range) { |
| 962 |        tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min); |
| 963 |        tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max); |
| 964 |        tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow; |
| 965 |        tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow; |
| 966 |      } else { |
| 967 |        tssa->var_info[ssa_var].has_range = 1; |
| 968 |        tssa->var_info[ssa_var].range = info->range; |
| 969 |      } |
| 970 |    } |
| 971 |    return 1; |
| 972 |  } |
| 973 |  return 0; |
| 974 | } |
| 975 | |
| 976 | static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op) |
| 977 | { |
| 978 |  while (1) { |
| 979 |    if (p->op == ZEND_JIT_TRACE_VM) { |
| 980 |      if (p->opline->opcode == ZEND_DO_UCALL |
| 981 |       || p->opline->opcode == ZEND_DO_FCALL_BY_NAME |
| 982 |       || p->opline->opcode == ZEND_DO_FCALL) { |
| 983 |        if (p->opline->result_type != IS_UNUSED) { |
| 984 |          return ssa_op->result_def; |
| 985 |        } |
| 986 |      } |
| 987 |      return -1; |
| 988 |    } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) { |
| 989 |      /*skip */ |
| 990 |    } else { |
| 991 |      return -1; |
| 992 |    } |
| 993 |    p--; |
| 994 |  } |
| 995 | } |
| 996 | |
| 997 | static const zend_op *zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec *p, const zend_op_array *op_array) |
| 998 | { |
| 999 |  if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { |
| 1000 |    p--; |
| 1001 |    while (1) { |
| 1002 |      if (p->op == ZEND_JIT_TRACE_VM) { |
| 1003 |        if (p->opline->opcode == ZEND_INIT_FCALL |
| 1004 |         || p->opline->opcode == ZEND_INIT_FCALL_BY_NAME |
| 1005 |         || p->opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME |
| 1006 |         || p->opline->opcode == ZEND_INIT_DYNAMIC_CALL |
| 1007 |         || p->opline->opcode == ZEND_INIT_USER_CALL |
| 1008 |         || p->opline->opcode == ZEND_NEW |
| 1009 |         || p->opline->opcode == ZEND_INIT_METHOD_CALL |
| 1010 |         || p->opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) { |
| 1011 |          return p->opline; |
| 1012 |        } |
| 1013 |        return NULL; |
| 1014 |      } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) { |
| 1015 |        /*skip */ |
| 1016 |      } else { |
| 1017 |        return NULL; |
| 1018 |      } |
| 1019 |      p--; |
| 1020 |    } |
| 1021 |  } else { |
| 1022 |    const zend_op *opline = NULL; |
| 1023 |    int call_level = 0; |
| 1024 | |
| 1025 |    p++; |
| 1026 |    while (1) { |
| 1027 |      if (p->op == ZEND_JIT_TRACE_VM) { |
| 1028 |        opline = p->opline; |
| 1029 |        break; |
| 1030 |      } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { |
| 1031 |        call_level++; |
| 1032 |        /*skip */ |
| 1033 |      } else { |
| 1034 |        return NULL; |
| 1035 |      } |
| 1036 |      p--; |
| 1037 |    } |
| 1038 |    if (opline) { |
| 1039 |      while (opline > op_array->opcodes) { |
| 1040 |        opline--; |
| 1041 |        switch (opline->opcode) { |
| 1042 |          case ZEND_INIT_FCALL: |
| 1043 |          case ZEND_INIT_FCALL_BY_NAME: |
| 1044 |          case ZEND_INIT_NS_FCALL_BY_NAME: |
| 1045 |          case ZEND_INIT_METHOD_CALL: |
| 1046 |          case ZEND_INIT_DYNAMIC_CALL: |
| 1047 |          case ZEND_INIT_STATIC_METHOD_CALL: |
| 1048 |          case ZEND_INIT_USER_CALL: |
| 1049 |          case ZEND_NEW: |
| 1050 |            if (call_level == 0) { |
| 1051 |              return opline; |
| 1052 |            } |
| 1053 |            call_level--; |
| 1054 |            break; |
| 1055 |          case ZEND_DO_FCALL: |
| 1056 |          case ZEND_DO_ICALL: |
| 1057 |          case ZEND_DO_UCALL: |
| 1058 |          case ZEND_DO_FCALL_BY_NAME: |
| 1059 |            call_level++; |
| 1060 |            break; |
| 1061 |        } |
| 1062 |      } |
| 1063 |    } |
| 1064 |  } |
| 1065 |  return NULL; |
| 1066 | } |
| 1067 | |
| 1068 | static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var) |
| 1069 | { |
| 1070 |  if ((tssa->var_info[phi_var].type & MAY_BE_ANY) == MAY_BE_LONG |
| 1071 |   && !(tssa->var_info[var].type & MAY_BE_REF)) { |
| 1072 |    int idx = tssa->vars[var].definition; |
| 1073 | |
| 1074 |    if (idx >= 0) { |
| 1075 |      if (tssa->ops[idx].op1_def == var) { |
| 1076 |        const zend_op *opline = ssa_opcodes[idx]; |
| 1077 |        if (opline->opcode == ZEND_PRE_DEC |
| 1078 |         || opline->opcode == ZEND_PRE_INC |
| 1079 |         || opline->opcode == ZEND_POST_DEC |
| 1080 |         || opline->opcode == ZEND_POST_INC) { |
| 1081 |          if (tssa->ops[idx].op1_use >= 0 |
| 1082 |           && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_STRING)) { |
| 1083 |            return 0; |
| 1084 |          } |
| 1085 |          return 1; |
| 1086 |        } else if (opline->opcode == ZEND_ASSIGN_OP |
| 1087 |         && (opline->extended_value == ZEND_ADD |
| 1088 |         || opline->extended_value == ZEND_SUB |
| 1089 |         || opline->extended_value == ZEND_MUL)) { |
| 1090 |          if ((opline->op2_type & (IS_VAR|IS_CV)) |
| 1091 |           && tssa->ops[idx].op2_use >= 0 |
| 1092 |           && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) { |
| 1093 |            return 0; |
| 1094 |          } |
| 1095 |          return 1; |
| 1096 |        } |
| 1097 |      } |
| 1098 |      if (tssa->ops[idx].result_def == var) { |
| 1099 |        const zend_op *opline = ssa_opcodes[idx]; |
| 1100 |        if (opline->opcode == ZEND_ADD |
| 1101 |         || opline->opcode == ZEND_SUB |
| 1102 |         || opline->opcode == ZEND_MUL |
| 1103 |         || opline->opcode == ZEND_PRE_DEC |
| 1104 |         || opline->opcode == ZEND_PRE_INC |
| 1105 |         || opline->opcode == ZEND_POST_DEC |
| 1106 |         || opline->opcode == ZEND_POST_INC) { |
| 1107 |          if ((opline->op1_type & (IS_VAR|IS_CV)) |
| 1108 |           && tssa->ops[idx].op1_use >= 0 |
| 1109 |           && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) { |
| 1110 |            return 0; |
| 1111 |          } |
| 1112 |          if ((opline->op2_type & (IS_VAR|IS_CV)) |
| 1113 |           && tssa->ops[idx].op2_use >= 0 |
| 1114 |           && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) { |
| 1115 |            return 0; |
| 1116 |          } |
| 1117 |          return 1; |
| 1118 |        } |
| 1119 |      } |
| 1120 |    } |
| 1121 |  } |
| 1122 |  return 0; |
| 1123 | } |
| 1124 | |
| 1125 | typedef struct _zend_tssa { |
| 1126 |  zend_ssa ssa; |
| 1127 |  const zend_op **tssa_opcodes; |
| 1128 |  int used_stack; |
| 1129 | } zend_tssa; |
| 1130 | |
| 1131 | static const zend_op _nop_opcode = {0}; |
| 1132 | |
| 1133 | static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr) |
| 1134 | { |
| 1135 |  zend_ssa *tssa; |
| 1136 |  zend_ssa_op *ssa_ops, *op; |
| 1137 |  zend_ssa_var *ssa_vars; |
| 1138 |  zend_ssa_var_info *ssa_var_info; |
| 1139 |  const zend_op_array *op_array; |
| 1140 |  const zend_op *opline; |
| 1141 |  const zend_op **ssa_opcodes; |
| 1142 |  zend_jit_trace_rec *p; |
| 1143 |  int i, v, idx, len, ssa_ops_count, vars_count, ssa_vars_count; |
| 1144 |  zend_jit_trace_stack *stack; |
| 1145 |  uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS; |
| 1146 |  uint32_t optimization_level = 0; |
| 1147 |  int call_level, level, num_op_arrays, used_stack, max_used_stack; |
| 1148 |  size_t frame_size, stack_top, stack_size, stack_bottom; |
| 1149 |  zend_jit_op_array_trace_extension *jit_extension; |
| 1150 |  zend_ssa *ssa; |
| 1151 |  zend_jit_trace_stack_frame *frame, *top, *call; |
| 1152 |  zend_ssa_var_info return_value_info; |
| 1153 | |
| 1154 |  /* 1. Count number of TSSA opcodes; |
| 1155 |   * Count number of activation frames; |
| 1156 |   * Calculate size of abstract stack; |
| 1157 |   * Construct regular SSA for involved op_array */ |
| 1158 |  op_array = trace_buffer->op_array; |
| 1159 |  stack_top = stack_size = zend_jit_trace_frame_size(op_array); |
| 1160 |  stack_bottom = 0; |
| 1161 |  p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; |
| 1162 |  ssa_ops_count = 0; |
| 1163 |  call_level = 0; |
| 1164 |  level = 0; |
| 1165 |  num_op_arrays = 0; |
| 1166 |  /* Remember op_array to cleanup */ |
| 1167 |  op_arrays[num_op_arrays++] = op_array; |
| 1168 |  /* Build SSA */ |
| 1169 |  ssa = zend_jit_trace_build_ssa(op_array, script); |
| 1170 |  for (;;p++) { |
| 1171 |    if (p->op == ZEND_JIT_TRACE_VM) { |
| 1172 |      if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) { |
| 1173 |        const zend_op *opline = p->opline; |
| 1174 | |
| 1175 |        switch (opline->opcode) { |
| 1176 |          case ZEND_INCLUDE_OR_EVAL: |
| 1177 |            ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS; |
| 1178 |            break; |
| 1179 |          case ZEND_FETCH_R: |
| 1180 |          case ZEND_FETCH_W: |
| 1181 |          case ZEND_FETCH_RW: |
| 1182 |          case ZEND_FETCH_FUNC_ARG: |
| 1183 |          case ZEND_FETCH_IS: |
| 1184 |          case ZEND_FETCH_UNSET: |
| 1185 |          case ZEND_UNSET_VAR: |
| 1186 |          case ZEND_ISSET_ISEMPTY_VAR: |
| 1187 |            if (opline->extended_value & ZEND_FETCH_LOCAL) { |
| 1188 |              ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS; |
| 1189 |            } else if ((opline->extended_value & (ZEND_FETCH_GLOBAL | ZEND_FETCH_GLOBAL_LOCK)) && |
| 1190 |             !op_array->function_name) { |
| 1191 |              ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS; |
| 1192 |            } |
| 1193 |            break; |
| 1194 |        } |
| 1195 |      } |
| 1196 |      ssa_ops_count += zend_jit_trace_op_len(p->opline); |
| 1197 |    } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { |
| 1198 |      call_level++; |
| 1199 |      stack_top += zend_jit_trace_frame_size(p->op_array); |
| 1200 |      if (stack_top > stack_size) { |
| 1201 |        stack_size = stack_top; |
| 1202 |      } |
| 1203 |    } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { |
| 1204 |      if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) { |
| 1205 |        if (p->func != (zend_function*)&zend_pass_function |
| 1206 |         && (zend_string_equals_literal(p->func->common.function_name, "extract") |
| 1207 |         || zend_string_equals_literal(p->func->common.function_name, "compact") |
| 1208 |         || zend_string_equals_literal(p->func->common.function_name, "get_defined_vars"))) { |
| 1209 |          ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS; |
| 1210 |        } |
| 1211 |      } |
| 1212 |      frame_size = zend_jit_trace_frame_size(p->op_array); |
| 1213 |      if (call_level == 0) { |
| 1214 |        if (stack_top + frame_size > stack_size) { |
| 1215 |          stack_size = stack_top + frame_size; |
| 1216 |        } |
| 1217 |      } else { |
| 1218 |        call_level--; |
| 1219 |        stack_top -= frame_size; |
| 1220 |      } |
| 1221 |    } else if (p->op == ZEND_JIT_TRACE_ENTER) { |
| 1222 |      op_array = p->op_array; |
| 1223 |      if (call_level == 0) { |
| 1224 |        stack_top += zend_jit_trace_frame_size(op_array); |
| 1225 |        if (stack_top > stack_size) { |
| 1226 |          stack_size = stack_top; |
| 1227 |        } |
| 1228 |      } else { |
| 1229 |        call_level--; |
| 1230 |      } |
| 1231 |      level++; |
| 1232 |      jit_extension = |
| 1233 |        (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 1234 |      ssa = &jit_extension->func_info.ssa; |
| 1235 |      if (ssa->cfg.blocks_count) { |
| 1236 |        /* pass */ |
| 1237 |      } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) { |
| 1238 |        /* Too many functions in single trace */ |
| 1239 |        *num_op_arrays_ptr = num_op_arrays; |
| 1240 |        return NULL; |
| 1241 |      } else { |
| 1242 |        /* Remember op_array to cleanup */ |
| 1243 |        op_arrays[num_op_arrays++] = op_array; |
| 1244 |        /* Build SSA */ |
| 1245 |        ssa = zend_jit_trace_build_ssa(op_array, script); |
| 1246 |      } |
| 1247 |    } else if (p->op == ZEND_JIT_TRACE_BACK) { |
| 1248 |      if (level == 0) { |
| 1249 |        stack_bottom += zend_jit_trace_frame_size(p->op_array); |
| 1250 |        jit_extension = |
| 1251 |          (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 1252 |        ssa = &jit_extension->func_info.ssa; |
| 1253 |        if (ssa->cfg.blocks_count) { |
| 1254 |          /* pass */ |
| 1255 |        } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) { |
| 1256 |          /* Too many functions in single trace */ |
| 1257 |          *num_op_arrays_ptr = num_op_arrays; |
| 1258 |          return NULL; |
| 1259 |        } else { |
| 1260 |          /* Remember op_array to cleanup */ |
| 1261 |          op_arrays[num_op_arrays++] = op_array; |
| 1262 |          /* Build SSA */ |
| 1263 |          ssa = zend_jit_trace_build_ssa(op_array, script); |
| 1264 |        } |
| 1265 |      } else { |
| 1266 |        stack_top -= zend_jit_trace_frame_size(op_array); |
| 1267 |        level--; |
| 1268 |      } |
| 1269 |      op_array = p->op_array; |
| 1270 |    } else if (p->op == ZEND_JIT_TRACE_END) { |
| 1271 |      break; |
| 1272 |    } |
| 1273 |  } |
| 1274 |  *num_op_arrays_ptr = num_op_arrays; |
| 1275 | |
| 1276 |  /* Allocate space for abstract stack */ |
| 1277 |  JIT_G(current_frame) = frame = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom); |
| 1278 | |
| 1279 |  /* 2. Construct TSSA */ |
| 1280 |  tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_tssa)); |
| 1281 |  tssa->cfg.flags = ZEND_SSA_TSSA; |
| 1282 |  tssa->cfg.blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_basic_block)); |
| 1283 |  tssa->blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_ssa_block)); |
| 1284 |  tssa->cfg.predecessors = zend_arena_calloc(&CG(arena), 2, sizeof(int)); |
| 1285 | |
| 1286 |  if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP |
| 1287 |   || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL |
| 1288 |   || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 1289 |    tssa->cfg.blocks_count = 2; |
| 1290 |    tssa->cfg.edges_count = 2; |
| 1291 | |
| 1292 |    tssa->cfg.predecessors[0] = 0; |
| 1293 |    tssa->cfg.predecessors[1] = 1; |
| 1294 | |
| 1295 |    tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_REACHABLE; |
| 1296 |    tssa->cfg.blocks[0].successors_count = 1; |
| 1297 |    tssa->cfg.blocks[0].predecessors_count = 0; |
| 1298 |    tssa->cfg.blocks[0].successors = tssa->cfg.blocks[0].successors_storage; |
| 1299 |    tssa->cfg.blocks[0].successors[0] = 1; |
| 1300 | |
| 1301 |    tssa->cfg.blocks[0].flags = ZEND_BB_FOLLOW|ZEND_BB_TARGET|ZEND_BB_LOOP_HEADER|ZEND_BB_REACHABLE; |
| 1302 |    tssa->cfg.blocks[1].successors_count = 1; |
| 1303 |    tssa->cfg.blocks[1].predecessors_count = 2; |
| 1304 |    tssa->cfg.blocks[1].successors = tssa->cfg.blocks[1].successors_storage; |
| 1305 |    tssa->cfg.blocks[1].successors[1] = 1; |
| 1306 |  } else { |
| 1307 |    tssa->cfg.blocks_count = 1; |
| 1308 |    tssa->cfg.edges_count = 0; |
| 1309 | |
| 1310 |    tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_EXIT|ZEND_BB_REACHABLE; |
| 1311 |    tssa->cfg.blocks[0].successors_count = 0; |
| 1312 |    tssa->cfg.blocks[0].predecessors_count = 0; |
| 1313 |  } |
| 1314 |  ((zend_tssa*)tssa)->used_stack = -1; |
| 1315 | |
| 1316 |  if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) { |
| 1317 |    return tssa; |
| 1318 |  } |
| 1319 | |
| 1320 |  tssa->ops = ssa_ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op)); |
| 1321 |  memset(ssa_ops, -1, ssa_ops_count * sizeof(zend_ssa_op)); |
| 1322 |  ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count + 1, sizeof(zend_op*)); |
| 1323 |  ((zend_tssa*)tssa)->tssa_opcodes = ssa_opcodes; |
| 1324 |  ssa_opcodes[ssa_ops_count] = &_nop_opcode; |
| 1325 | |
| 1326 |  op_array = trace_buffer->op_array; |
| 1327 |  if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { |
| 1328 |    ssa_vars_count = op_array->last_var; |
| 1329 |  } else { |
| 1330 |    ssa_vars_count = op_array->last_var + op_array->T; |
| 1331 |  } |
| 1332 |  stack = frame->stack; |
| 1333 |  for (i = 0; i < ssa_vars_count; i++) { |
| 1334 |    SET_STACK_VAR(stack, i, i); |
| 1335 |  } |
| 1336 | |
| 1337 |  if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) { |
| 1338 |    // TODO: For tracing, it's possible, to create pseudo Phi functions |
| 1339 |    // at the end of loop, without this additional pass (like LuaJIT) ??? |
| 1340 |    ssa_vars_count = zend_jit_trace_add_phis(trace_buffer, ssa_vars_count, tssa, stack); |
| 1341 |  } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) { |
| 1342 |    ssa_vars_count = zend_jit_trace_add_call_phis(trace_buffer, ssa_vars_count, tssa, stack); |
| 1343 |  } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 1344 |    ssa_vars_count = zend_jit_trace_add_ret_phis(trace_buffer, ssa_vars_count, tssa, stack); |
| 1345 |  } |
| 1346 | |
| 1347 |  p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; |
| 1348 |  idx = 0; |
| 1349 |  level = 0; |
| 1350 |  for (;;p++) { |
| 1351 |    if (p->op == ZEND_JIT_TRACE_VM) { |
| 1352 |      opline = p->opline; |
| 1353 |      ssa_opcodes[idx] = opline; |
| 1354 |      ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack); |
| 1355 |      idx++; |
| 1356 |      len = zend_jit_trace_op_len(p->opline); |
| 1357 |      while (len > 1) { |
| 1358 |        opline++; |
| 1359 |        ssa_opcodes[idx] = opline; |
| 1360 |        if (opline->opcode != ZEND_OP_DATA) { |
| 1361 |          ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack); |
| 1362 |        } |
| 1363 |        idx++; |
| 1364 |        len--; |
| 1365 |      } |
| 1366 |    } else if (p->op == ZEND_JIT_TRACE_ENTER) { |
| 1367 |      frame = zend_jit_trace_call_frame(frame, op_array); |
| 1368 |      stack = frame->stack; |
| 1369 |      op_array = p->op_array; |
| 1370 |      level++; |
| 1371 |      if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) { |
| 1372 |        return NULL; |
| 1373 |      } |
| 1374 |      ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count); |
| 1375 |      for (i = 0; i < op_array->last_var; i++) { |
| 1376 |        SET_STACK_VAR(stack, i, ssa_vars_count++); |
| 1377 |      } |
| 1378 |    } else if (p->op == ZEND_JIT_TRACE_BACK) { |
| 1379 |      op_array = p->op_array; |
| 1380 |      frame = zend_jit_trace_ret_frame(frame, op_array); |
| 1381 |      stack = frame->stack; |
| 1382 |      if (level == 0) { |
| 1383 |        if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) { |
| 1384 |          return NULL; |
| 1385 |        } |
| 1386 |        ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count); |
| 1387 |        for (i = 0; i < op_array->last_var + op_array->T; i++) { |
| 1388 |          SET_STACK_VAR(stack, i, ssa_vars_count++); |
| 1389 |        } |
| 1390 |      } else { |
| 1391 |        level--; |
| 1392 |      } |
| 1393 |    } else if (p->op == ZEND_JIT_TRACE_END) { |
| 1394 |      break; |
| 1395 |    } |
| 1396 |  } |
| 1397 | |
| 1398 |  op_array = trace_buffer->op_array; |
| 1399 |  tssa->vars_count = ssa_vars_count; |
| 1400 |  tssa->vars = ssa_vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var)); |
| 1401 |  if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { |
| 1402 |    vars_count = op_array->last_var; |
| 1403 |  } else { |
| 1404 |    vars_count = op_array->last_var + op_array->T; |
| 1405 |  } |
| 1406 |  i = 0; |
| 1407 |  while (i < vars_count) { |
| 1408 |    ssa_vars[i].var = i; |
| 1409 |    ssa_vars[i].scc = -1; |
| 1410 |    ssa_vars[i].definition = -1; |
| 1411 |    ssa_vars[i].use_chain = -1; |
| 1412 |    i++; |
| 1413 |  } |
| 1414 |  while (i < tssa->vars_count) { |
| 1415 |    ssa_vars[i].var = -1; |
| 1416 |    ssa_vars[i].scc = -1; |
| 1417 |    ssa_vars[i].definition = -1; |
| 1418 |    ssa_vars[i].use_chain = -1; |
| 1419 |    i++; |
| 1420 |  } |
| 1421 | |
| 1422 |  if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP |
| 1423 |   || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL |
| 1424 |   || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 1425 |    /* Update Phi sources */ |
| 1426 |    zend_ssa_phi *phi = tssa->blocks[1].phis; |
| 1427 | |
| 1428 |    while (phi) { |
| 1429 |      phi->sources[1] = STACK_VAR(stack, phi->var); |
| 1430 |      ssa_vars[phi->ssa_var].var = phi->var; |
| 1431 |      ssa_vars[phi->ssa_var].definition_phi = phi; |
| 1432 |      ssa_vars[phi->sources[0]].phi_use_chain = phi; |
| 1433 |      ssa_vars[phi->sources[1]].phi_use_chain = phi; |
| 1434 |      phi = phi->next; |
| 1435 |    } |
| 1436 |  } |
| 1437 | |
| 1438 |  /* 3. Compute use-def chains */ |
| 1439 |  idx = (ssa_ops_count - 1); |
| 1440 |  op = ssa_ops + idx; |
| 1441 |  while (idx >= 0) { |
| 1442 |    opline = ssa_opcodes[idx]; |
| 1443 |    if (op->op1_use >= 0) { |
| 1444 |      op->op1_use_chain = ssa_vars[op->op1_use].use_chain; |
| 1445 |      ssa_vars[op->op1_use].use_chain = idx; |
| 1446 |    } |
| 1447 |    if (op->op2_use >= 0 && op->op2_use != op->op1_use) { |
| 1448 |      op->op2_use_chain = ssa_vars[op->op2_use].use_chain; |
| 1449 |      ssa_vars[op->op2_use].use_chain = idx; |
| 1450 |    } |
| 1451 |    if (op->result_use >= 0 && op->result_use != op->op1_use && op->result_use != op->op2_use) { |
| 1452 |      op->res_use_chain = ssa_vars[op->result_use].use_chain; |
| 1453 |      ssa_vars[op->result_use].use_chain = idx; |
| 1454 |    } |
| 1455 |    if (op->op1_def >= 0) { |
| 1456 |      ssa_vars[op->op1_def].var = EX_VAR_TO_NUM(opline->op1.var); |
| 1457 |      ssa_vars[op->op1_def].definition = idx; |
| 1458 |    } |
| 1459 |    if (op->op2_def >= 0) { |
| 1460 |      ssa_vars[op->op2_def].var = EX_VAR_TO_NUM(opline->op2.var); |
| 1461 |      ssa_vars[op->op2_def].definition = idx; |
| 1462 |    } |
| 1463 |    if (op->result_def >= 0) { |
| 1464 |      ssa_vars[op->result_def].var = EX_VAR_TO_NUM(opline->result.var); |
| 1465 |      ssa_vars[op->result_def].definition = idx; |
| 1466 |    } |
| 1467 |    op--; |
| 1468 |    idx--; |
| 1469 |  } |
| 1470 | |
| 1471 |  /* 4. Type inference */ |
| 1472 |  op_array = trace_buffer->op_array; |
| 1473 |  jit_extension = |
| 1474 |    (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 1475 |  ssa = &jit_extension->func_info.ssa; |
| 1476 | |
| 1477 |  tssa->var_info = ssa_var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info)); |
| 1478 | |
| 1479 |  if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { |
| 1480 |    i = 0; |
| 1481 |    while (i < op_array->last_var) { |
| 1482 |      if (i < op_array->num_args) { |
| 1483 |        if (ssa->var_info |
| 1484 |         && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) { |
| 1485 |          /* pass */ |
| 1486 |        } else { |
| 1487 |          if (ssa->vars) { |
| 1488 |            ssa_vars[i].no_val = ssa->vars[i].no_val; |
| 1489 |            ssa_vars[i].alias = ssa->vars[i].alias; |
| 1490 |          } else { |
| 1491 |            ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i); |
| 1492 |          } |
| 1493 |          if (op_array->arg_info) { |
| 1494 |            zend_arg_info *arg_info = &op_array->arg_info[i]; |
| 1495 |            zend_class_entry *ce; |
| 1496 |            uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce); |
| 1497 | |
| 1498 |            if (ZEND_ARG_SEND_MODE(arg_info)) { |
| 1499 |              tmp |= MAY_BE_REF; |
| 1500 |            } |
| 1501 |            ssa_var_info[i].type = tmp; |
| 1502 |            ssa_var_info[i].ce = ce; |
| 1503 |            ssa_var_info[i].is_instanceof = 1; |
| 1504 |          } else { |
| 1505 |            ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; |
| 1506 |          } |
| 1507 |        } |
| 1508 |      } else { |
| 1509 |        if (ssa->vars) { |
| 1510 |          ssa_vars[i].no_val = ssa->vars[i].no_val; |
| 1511 |          ssa_vars[i].alias = ssa->vars[i].alias; |
| 1512 |        } else { |
| 1513 |          ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i); |
| 1514 |        } |
| 1515 |        if (ssa_vars[i].alias == NO_ALIAS) { |
| 1516 |          ssa_var_info[i].type = MAY_BE_UNDEF; |
| 1517 |        } else { |
| 1518 |          ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; |
| 1519 |        } |
| 1520 |      } |
| 1521 |      i++; |
| 1522 |    } |
| 1523 |  } else { |
| 1524 |    int parent_vars_count = 0; |
| 1525 |    zend_jit_trace_stack *parent_stack = NULL; |
| 1526 | |
| 1527 |    i = 0; |
| 1528 |    if (parent_trace) { |
| 1529 |      parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size, |
| 1530 |        op_array->last_var + op_array->T); |
| 1531 |      if (parent_vars_count) { |
| 1532 |        parent_stack = |
| 1533 |          zend_jit_traces[parent_trace].stack_map + |
| 1534 |          zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset; |
| 1535 |      } |
| 1536 |    } |
| 1537 |    while (i < op_array->last_var + op_array->T) { |
| 1538 |      if (!ssa->var_info |
| 1539 |       || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) { |
| 1540 |        if (ssa->vars && i < ssa->vars_count) { |
| 1541 |          ssa_vars[i].alias = ssa->vars[i].alias; |
| 1542 |        } else { |
| 1543 |          ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i); |
| 1544 |        } |
| 1545 |        if (i < op_array->last_var) { |
| 1546 |          ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; |
| 1547 |        } else { |
| 1548 |          ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; |
| 1549 |        } |
| 1550 |      } |
| 1551 |      if (i < parent_vars_count) { |
| 1552 |        /* Initialize TSSA variable from parent trace */ |
| 1553 |        zend_uchar op_type = STACK_TYPE(parent_stack, i); |
| 1554 | |
| 1555 |        if (op_type != IS_UNKNOWN) { |
| 1556 |          ssa_var_info[i].type &= zend_jit_trace_type_to_info(op_type); |
| 1557 |          if (!ssa_var_info[i].type |
| 1558 |           && op_type == IS_UNDEF |
| 1559 |           && i >= op_array->last_var) { |
| 1560 |            // TODO: It's better to use NULL instead of UNDEF for temporary variables |
| 1561 |            ssa_var_info[i].type |= MAY_BE_UNDEF; |
| 1562 |          } |
| 1563 |        } |
| 1564 |      } |
| 1565 |      i++; |
| 1566 |    } |
| 1567 |  } |
| 1568 | |
| 1569 |  if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP |
| 1570 |   || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL |
| 1571 |   || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 1572 |    /* Propagate initial value through Phi functions */ |
| 1573 |    zend_ssa_phi *phi = tssa->blocks[1].phis; |
| 1574 | |
| 1575 |    while (phi) { |
| 1576 |      if (!ssa->var_info |
| 1577 |       || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, phi->ssa_var)) { |
| 1578 |        ssa_vars[phi->ssa_var].alias = ssa_vars[phi->sources[0]].alias; |
| 1579 |        ssa_var_info[phi->ssa_var].type = ssa_var_info[phi->sources[0]].type; |
| 1580 |      } |
| 1581 |      phi = phi->next; |
| 1582 |    } |
| 1583 |  } |
| 1584 | |
| 1585 |  frame = JIT_G(current_frame); |
| 1586 |  top = zend_jit_trace_call_frame(frame, op_array); |
| 1587 |  TRACE_FRAME_INIT(frame, op_array, 0, 0); |
| 1588 |  TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1); |
| 1589 |  frame->used_stack = 0; |
| 1590 |  for (i = 0; i < op_array->last_var + op_array->T; i++) { |
| 1591 |    SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1); |
| 1592 |  } |
| 1593 |  memset(&return_value_info, 0, sizeof(return_value_info)); |
| 1594 | |
| 1595 |  if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) { |
| 1596 |    max_used_stack = used_stack = 0; |
| 1597 |  } else { |
| 1598 |    max_used_stack = used_stack = -1; |
| 1599 |  } |
| 1600 | |
| 1601 |  p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; |
| 1602 |  idx = 0; |
| 1603 |  level = 0; |
| 1604 |  opline = NULL; |
| 1605 |  for (;;p++) { |
| 1606 |    if (p->op == ZEND_JIT_TRACE_VM) { |
| 1607 |      uint8_t orig_op1_type, orig_op2_type, op1_type, op2_type, op3_type; |
| 1608 |      uint8_t val_type = IS_UNKNOWN; |
| 1609 | //      zend_class_entry *op1_ce = NULL; |
| 1610 |      zend_class_entry *op2_ce = NULL; |
| 1611 | |
| 1612 |      opline = p->opline; |
| 1613 | |
| 1614 |      op1_type = orig_op1_type = p->op1_type; |
| 1615 |      op2_type = orig_op2_type = p->op2_type; |
| 1616 |      op3_type = p->op3_type; |
| 1617 |      if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) { |
| 1618 |        op1_type = IS_UNKNOWN; |
| 1619 |      } |
| 1620 |      if (op1_type != IS_UNKNOWN) { |
| 1621 |        op1_type &= ~IS_TRACE_PACKED; |
| 1622 |      } |
| 1623 |      if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) { |
| 1624 |        op2_type = IS_UNKNOWN; |
| 1625 |      } |
| 1626 |      if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) { |
| 1627 |        op3_type = IS_UNKNOWN; |
| 1628 |      } |
| 1629 | |
| 1630 |      if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) { |
| 1631 | //        op1_ce = (zend_class_entry*)(p+1)->ce; |
| 1632 |        p++; |
| 1633 |      } |
| 1634 |      if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) { |
| 1635 |        op2_ce = (zend_class_entry*)(p+1)->ce; |
| 1636 |        p++; |
| 1637 |      } |
| 1638 |      if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) { |
| 1639 |        val_type = (p+1)->op1_type; |
| 1640 |        p++; |
| 1641 |      } |
| 1642 | |
| 1643 |      switch (opline->opcode) { |
| 1644 |        case ZEND_ASSIGN_OP: |
| 1645 |          if (opline->extended_value == ZEND_POW |
| 1646 |           || opline->extended_value == ZEND_DIV) { |
| 1647 |            // TODO: check for division by zero ??? |
| 1648 |            break; |
| 1649 |          } |
| 1650 |          if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) { |
| 1651 |            break; |
| 1652 |          } |
| 1653 |          ADD_OP1_TRACE_GUARD(); |
| 1654 |          ADD_OP2_TRACE_GUARD(); |
| 1655 |          break; |
| 1656 |        case ZEND_ASSIGN_DIM_OP: |
| 1657 |          if (opline->extended_value == ZEND_POW |
| 1658 |           || opline->extended_value == ZEND_DIV) { |
| 1659 |            // TODO: check for division by zero ??? |
| 1660 |            break; |
| 1661 |          } |
| 1662 |          if (opline->result_type != IS_UNUSED) { |
| 1663 |            break; |
| 1664 |          } |
| 1665 |          if (op3_type != IS_UNKNOWN |
| 1666 |           && !zend_jit_supported_binary_op( |
| 1667 |              opline->extended_value, MAY_BE_ANY, (1<<op3_type))) { |
| 1668 |            break; |
| 1669 |          } |
| 1670 |          ZEND_FALLTHROUGH; |
| 1671 |        case ZEND_ASSIGN_DIM: |
| 1672 |          if (opline->op1_type == IS_CV) { |
| 1673 |            if ((opline+1)->op1_type == IS_CV |
| 1674 |             && ) { |
| 1675 |              /* skip $a[x] = $a; */ |
| 1676 |              break; |
| 1677 |            } |
| 1678 |            ADD_OP1_DATA_TRACE_GUARD(); |
| 1679 |            ADD_OP2_TRACE_GUARD(); |
| 1680 |            ADD_OP1_TRACE_GUARD(); |
| 1681 |          } else if ( |
| 1682 |           |
| 1683 |           ) { |
| 1684 |            if (opline->opcode == ZEND_ASSIGN_DIM_OP) { |
| 1685 |              ADD_OP1_DATA_TRACE_GUARD(); |
| 1686 |            } |
| 1687 |            ADD_OP2_TRACE_GUARD(); |
| 1688 |          } |
| 1689 |          if (op1_type == IS_ARRAY |
| 1690 |           && ((opline->op2_type == IS_CONST |
| 1691 |           && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG) |
| 1692 |           || (opline->op2_type != IS_CONST |
| 1693 |           && op2_type == IS_LONG))) { |
| 1694 | |
| 1695 |            if (!(orig_op1_type & IS_TRACE_PACKED)) { |
| 1696 |              zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use]; |
| 1697 | |
| 1698 |              if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) { |
| 1699 |                info->type |= MAY_BE_PACKED_GUARD; |
| 1700 |                info->type &= ~MAY_BE_ARRAY_PACKED; |
| 1701 |              } |
| 1702 |            } else if (opline->opcode == ZEND_ASSIGN_DIM_OP |
| 1703 |                && val_type != IS_UNKNOWN |
| 1704 |                && val_type != IS_UNDEF) { |
| 1705 |              zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use]; |
| 1706 | |
| 1707 |              if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) { |
| 1708 |                info->type |= MAY_BE_PACKED_GUARD; |
| 1709 |                info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH); |
| 1710 |              } |
| 1711 |            } |
| 1712 |          } |
| 1713 |          break; |
| 1714 |        case ZEND_ASSIGN_OBJ_OP: |
| 1715 |          if (opline->extended_value == ZEND_POW |
| 1716 |           || opline->extended_value == ZEND_DIV) { |
| 1717 |            // TODO: check for division by zero ??? |
| 1718 |            break; |
| 1719 |          } |
| 1720 |          if (opline->result_type != IS_UNUSED) { |
| 1721 |            break; |
| 1722 |          } |
| 1723 |          ZEND_FALLTHROUGH; |
| 1724 |        case ZEND_ASSIGN_OBJ: |
| 1725 |        case ZEND_PRE_INC_OBJ: |
| 1726 |        case ZEND_PRE_DEC_OBJ: |
| 1727 |        case ZEND_POST_INC_OBJ: |
| 1728 |        case ZEND_POST_DEC_OBJ: |
| 1729 |          if (opline->op2_type != IS_CONST |
| 1730 |           || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING |
| 1731 |           || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') { |
| 1732 |            break; |
| 1733 |          } |
| 1734 |          if (opline->opcode == ZEND_ASSIGN_OBJ_OP) { |
| 1735 |            if (opline->op1_type == IS_CV |
| 1736 |             && (opline+1)->op1_type == IS_CV |
| 1737 |             && ) { |
| 1738 |              /* skip $a->prop += $a; */ |
| 1739 |              break; |
| 1740 |            } |
| 1741 |            ADD_OP1_DATA_TRACE_GUARD(); |
| 1742 |          } |
| 1743 |          ADD_OP1_TRACE_GUARD(); |
| 1744 |          break; |
| 1745 |        case ZEND_CONCAT: |
| 1746 |        case ZEND_FAST_CONCAT: |
| 1747 |          if ((opline->op1_type == IS_CONST || orig_op1_type == IS_STRING) |
| 1748 |           && (opline->op2_type == IS_CONST || orig_op2_type == IS_STRING)) { |
| 1749 |            ADD_OP2_TRACE_GUARD(); |
| 1750 |            ADD_OP1_TRACE_GUARD(); |
| 1751 |          } |
| 1752 |          break; |
| 1753 |        case ZEND_IS_EQUAL: |
| 1754 |        case ZEND_IS_NOT_EQUAL: |
| 1755 |        case ZEND_IS_SMALLER: |
| 1756 |        case ZEND_IS_SMALLER_OR_EQUAL: |
| 1757 |        case ZEND_CASE: |
| 1758 |        case ZEND_IS_IDENTICAL: |
| 1759 |        case ZEND_IS_NOT_IDENTICAL: |
| 1760 |        case ZEND_CASE_STRICT: |
| 1761 |        case ZEND_BW_OR: |
| 1762 |        case ZEND_BW_AND: |
| 1763 |        case ZEND_BW_XOR: |
| 1764 |        case ZEND_SL: |
| 1765 |        case ZEND_SR: |
| 1766 |        case ZEND_MOD: |
| 1767 |        case ZEND_ADD: |
| 1768 |        case ZEND_SUB: |
| 1769 |        case ZEND_MUL: |
| 1770 | //        case ZEND_DIV: // TODO: check for division by zero ??? |
| 1771 |          ADD_OP2_TRACE_GUARD(); |
| 1772 |          ZEND_FALLTHROUGH; |
| 1773 |        case ZEND_ECHO: |
| 1774 |        case ZEND_STRLEN: |
| 1775 |        case ZEND_COUNT: |
| 1776 |        case ZEND_QM_ASSIGN: |
| 1777 |        case ZEND_FE_RESET_R: |
| 1778 |          ADD_OP1_TRACE_GUARD(); |
| 1779 |          break; |
| 1780 |        case ZEND_FE_FETCH_R: |
| 1781 |          ADD_OP1_TRACE_GUARD(); |
| 1782 |          if (op1_type == IS_ARRAY && (orig_op1_type & ~IS_TRACE_PACKED) == IS_ARRAY) { |
| 1783 | |
| 1784 |            zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use]; |
| 1785 | |
| 1786 |            if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) { |
| 1787 |              info->type |= MAY_BE_PACKED_GUARD; |
| 1788 |              if (orig_op1_type & IS_TRACE_PACKED) { |
| 1789 |                info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH); |
| 1790 |              } else { |
| 1791 |                info->type &= ~MAY_BE_ARRAY_PACKED; |
| 1792 |              } |
| 1793 |            } |
| 1794 |          } |
| 1795 |          break; |
| 1796 |        case ZEND_VERIFY_RETURN_TYPE: |
| 1797 |          if (opline->op1_type == IS_UNUSED) { |
| 1798 |            /* Always throws */ |
| 1799 |            break; |
| 1800 |          } |
| 1801 |          if (opline->op1_type == IS_CONST) { |
| 1802 |            /* TODO Different instruction format, has return value */ |
| 1803 |            break; |
| 1804 |          } |
| 1805 |          if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) { |
| 1806 |            /* Not worth bothering with */ |
| 1807 |            break; |
| 1808 |          } |
| 1809 |          ADD_OP1_TRACE_GUARD(); |
| 1810 |          break; |
| 1811 |        case ZEND_FETCH_DIM_FUNC_ARG: |
| 1812 |          if (!frame |
| 1813 |           || !frame->call |
| 1814 |           || !frame->call->func |
| 1815 |           || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) { |
| 1816 |            break; |
| 1817 |          } |
| 1818 |          ADD_OP2_TRACE_GUARD(); |
| 1819 |          ADD_OP1_TRACE_GUARD(); |
| 1820 |          break; |
| 1821 |        case ZEND_PRE_INC: |
| 1822 |        case ZEND_PRE_DEC: |
| 1823 |        case ZEND_POST_INC: |
| 1824 |        case ZEND_POST_DEC: |
| 1825 |          if (opline->op1_type != IS_CV) { |
| 1826 |            break; |
| 1827 |          } |
| 1828 |          ADD_OP1_TRACE_GUARD(); |
| 1829 |          break; |
| 1830 |        case ZEND_ASSIGN: |
| 1831 |          if (opline->op1_type != IS_CV) { |
| 1832 |            break; |
| 1833 |          } |
| 1834 |          ADD_OP2_TRACE_GUARD(); |
| 1835 |          if (op1_type != IS_UNKNOWN |
| 1836 |           && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) { |
| 1837 |            ADD_OP1_TRACE_GUARD(); |
| 1838 |          } |
| 1839 |          break; |
| 1840 |        case ZEND_CAST: |
| 1841 |          if (opline->extended_value != op1_type) { |
| 1842 |            break; |
| 1843 |          } |
| 1844 |          ADD_OP1_TRACE_GUARD(); |
| 1845 |          break; |
| 1846 |        case ZEND_JMPZ: |
| 1847 |        case ZEND_JMPNZ: |
| 1848 |        case ZEND_JMPZ_EX: |
| 1849 |        case ZEND_JMPNZ_EX: |
| 1850 |        case ZEND_BOOL: |
| 1851 |        case ZEND_BOOL_NOT: |
| 1852 |          ADD_OP1_TRACE_GUARD(); |
| 1853 |          break; |
| 1854 |        case ZEND_ISSET_ISEMPTY_CV: |
| 1855 |          if ((opline->extended_value & ZEND_ISEMPTY)) { |
| 1856 |            // TODO: support for empty() ??? |
| 1857 |            break; |
| 1858 |          } |
| 1859 |          ADD_OP1_TRACE_GUARD(); |
| 1860 |          break; |
| 1861 |        case ZEND_IN_ARRAY: |
| 1862 |          if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) { |
| 1863 |            break; |
| 1864 |          } |
| 1865 |          ADD_OP1_TRACE_GUARD(); |
| 1866 |          break; |
| 1867 |        case ZEND_ISSET_ISEMPTY_DIM_OBJ: |
| 1868 |          if ((opline->extended_value & ZEND_ISEMPTY)) { |
| 1869 |            // TODO: support for empty() ??? |
| 1870 |            break; |
| 1871 |          } |
| 1872 |          ZEND_FALLTHROUGH; |
| 1873 |        case ZEND_FETCH_DIM_R: |
| 1874 |        case ZEND_FETCH_DIM_IS: |
| 1875 |        case ZEND_FETCH_LIST_R: |
| 1876 |          ADD_OP1_TRACE_GUARD(); |
| 1877 |          ADD_OP2_TRACE_GUARD(); |
| 1878 | |
| 1879 |          if (op1_type == IS_ARRAY |
| 1880 |           && opline->op1_type != IS_CONST |
| 1881 |           && ((opline->op2_type == IS_CONST |
| 1882 |           && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG) |
| 1883 |           || (opline->op2_type != IS_CONST |
| 1884 |           && op2_type == IS_LONG))) { |
| 1885 | |
| 1886 |            zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use]; |
| 1887 | |
| 1888 |            if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) { |
| 1889 |              info->type |= MAY_BE_PACKED_GUARD; |
| 1890 |              if (orig_op1_type & IS_TRACE_PACKED) { |
| 1891 |                info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH); |
| 1892 |              } else { |
| 1893 |                info->type &= ~MAY_BE_ARRAY_PACKED; |
| 1894 |              } |
| 1895 |            } |
| 1896 |          } |
| 1897 |          break; |
| 1898 |        case ZEND_FETCH_DIM_W: |
| 1899 |        case ZEND_FETCH_DIM_RW: |
| 1900 | //        case ZEND_FETCH_DIM_UNSET: |
| 1901 |        case ZEND_FETCH_LIST_W: |
| 1902 |          if (opline->op1_type != IS_CV |
| 1903 |           && (orig_op1_type == IS_UNKNOWN |
| 1904 |           || !(orig_op1_type & IS_TRACE_INDIRECT))) { |
| 1905 |            break; |
| 1906 |          } |
| 1907 |          ADD_OP1_TRACE_GUARD(); |
| 1908 |          ADD_OP2_TRACE_GUARD(); |
| 1909 |          if (op1_type == IS_ARRAY |
| 1910 |           && !(orig_op1_type & IS_TRACE_PACKED) |
| 1911 |           && ((opline->op2_type == IS_CONST |
| 1912 |           && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG) |
| 1913 |           || (opline->op2_type != IS_CONST |
| 1914 |           && op2_type == IS_LONG))) { |
| 1915 | |
| 1916 |            zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use]; |
| 1917 | |
| 1918 |            if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) { |
| 1919 |              info->type |= MAY_BE_PACKED_GUARD; |
| 1920 |              info->type &= ~MAY_BE_ARRAY_PACKED; |
| 1921 |            } |
| 1922 |          } |
| 1923 |          break; |
| 1924 |        case ZEND_SEND_VAL_EX: |
| 1925 |        case ZEND_SEND_VAR_EX: |
| 1926 |        case ZEND_SEND_VAR_NO_REF_EX: |
| 1927 |          if (opline->op2_type == IS_CONST) { |
| 1928 |            /* Named parameters not supported in JIT */ |
| 1929 |            break; |
| 1930 |          } |
| 1931 |          if (opline->op2.num > MAX_ARG_FLAG_NUM) { |
| 1932 |            goto propagate_arg; |
| 1933 |          } |
| 1934 |          ZEND_FALLTHROUGH; |
| 1935 |        case ZEND_SEND_VAL: |
| 1936 |        case ZEND_SEND_VAR: |
| 1937 |        case ZEND_SEND_VAR_NO_REF: |
| 1938 |        case ZEND_SEND_FUNC_ARG: |
| 1939 |          if (opline->op2_type == IS_CONST) { |
| 1940 |            /* Named parameters not supported in JIT */ |
| 1941 |            break; |
| 1942 |          } |
| 1943 |          ADD_OP1_TRACE_GUARD(); |
| 1944 | propagate_arg: |
| 1945 |          /* Propagate argument type */ |
| 1946 |          if ( |
| 1947 |           |
| 1948 |           |
| 1949 |           ) { |
| 1950 |            uint32_t info; |
| 1951 | |
| 1952 |            if (opline->op1_type == IS_CONST) { |
| 1953 |              info = _const_op_type(RT_CONSTANT(opline, opline->op1)); |
| 1954 |            } else { |
| 1955 |              ZEND_ASSERT(ssa_ops[idx].op1_use >= 0); |
| 1956 |              info = ssa_var_info[ssa_ops[idx].op1_use].type & ~MAY_BE_GUARD; |
| 1957 |            } |
| 1958 |            if (frame->call->func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { |
| 1959 |              zend_arg_info *arg_info; |
| 1960 | |
| 1961 |              ZEND_ASSERT(frame->call->func->op_array.arg_info); |
| 1962 |              arg_info = &frame->call->func->op_array.arg_info[opline->op2.num - 1]; |
| 1963 |              if (ZEND_TYPE_IS_SET(arg_info->type)) { |
| 1964 |                zend_class_entry *ce; |
| 1965 |                uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce); |
| 1966 |                info &= tmp; |
| 1967 |                if (!info) { |
| 1968 |                  break; |
| 1969 |                } |
| 1970 |              } |
| 1971 |            } |
| 1972 |            if (opline->op1_type == IS_CV && (info & MAY_BE_RC1)) { |
| 1973 |              info |= MAY_BE_RCN; |
| 1974 |            } |
| 1975 |            if (info & MAY_BE_UNDEF) { |
| 1976 |              info |= MAY_BE_NULL; |
| 1977 |              info &= ~MAY_BE_UNDEF; |
| 1978 |            } |
| 1979 |            if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) { |
| 1980 |              info |= MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_KEY_ANY; |
| 1981 |            } |
| 1982 |            SET_STACK_INFO(frame->call->stack, opline->op2.num - 1, info); |
| 1983 |          } |
| 1984 |          break; |
| 1985 |        case ZEND_RETURN: |
| 1986 |          ADD_OP1_TRACE_GUARD(); |
| 1987 |          /* Propagate return value types */ |
| 1988 |          if (opline->op1_type == IS_UNUSED) { |
| 1989 |            return_value_info.type = MAY_BE_NULL; |
| 1990 |          } else if (opline->op1_type == IS_CONST) { |
| 1991 |            return_value_info.type = _const_op_type(RT_CONSTANT(opline, opline->op1)); |
| 1992 |          } else { |
| 1993 |            ZEND_ASSERT(ssa_ops[idx].op1_use >= 0); |
| 1994 |            return_value_info = ssa_var_info[ssa_ops[idx].op1_use]; |
| 1995 |            if (return_value_info.type & MAY_BE_UNDEF) { |
| 1996 |              return_value_info.type &= ~MAY_BE_UNDEF; |
| 1997 |              return_value_info.type |= MAY_BE_NULL; |
| 1998 |            } |
| 1999 |            if (return_value_info.type & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { |
| 2000 |              /* CVs are going to be destructed and the reference-counter |
| 2001 |               of return value may be decremented to 1 */ |
| 2002 |              return_value_info.type |= MAY_BE_RC1; |
| 2003 |            } |
| 2004 |            return_value_info.type &= ~MAY_BE_GUARD; |
| 2005 |          } |
| 2006 |          break; |
| 2007 |        case ZEND_CHECK_FUNC_ARG: |
| 2008 |          if (!frame |
| 2009 |           || !frame->call |
| 2010 |           || !frame->call->func) { |
| 2011 |            break; |
| 2012 |          } |
| 2013 |          if ( |
| 2014 |           ) { |
| 2015 |            /* Named parameters not supported in JIT */ |
| 2016 |            TRACE_FRAME_SET_LAST_SEND_UNKNOWN(frame->call); |
| 2017 |            break; |
| 2018 |          } |
| 2019 |          if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) { |
| 2020 |            TRACE_FRAME_SET_LAST_SEND_BY_REF(frame->call); |
| 2021 |          } else { |
| 2022 |            TRACE_FRAME_SET_LAST_SEND_BY_VAL(frame->call); |
| 2023 |          } |
| 2024 |          break; |
| 2025 |        case ZEND_FETCH_OBJ_FUNC_ARG: |
| 2026 |          if (!frame |
| 2027 |           || !frame->call |
| 2028 |           || !frame->call->func |
| 2029 |           || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) { |
| 2030 |            break; |
| 2031 |          } |
| 2032 |          ZEND_FALLTHROUGH; |
| 2033 |        case ZEND_FETCH_OBJ_R: |
| 2034 |        case ZEND_FETCH_OBJ_IS: |
| 2035 |        case ZEND_FETCH_OBJ_W: |
| 2036 |          if (opline->op2_type != IS_CONST |
| 2037 |           || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING |
| 2038 |           || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') { |
| 2039 |            break; |
| 2040 |          } |
| 2041 |          if (opline->op1_type != IS_UNUSED && op1_type == IS_OBJECT) { |
| 2042 |            ADD_OP1_TRACE_GUARD(); |
| 2043 |          } |
| 2044 |          break; |
| 2045 |        case ZEND_INIT_METHOD_CALL: |
| 2046 |          if (opline->op2_type != IS_CONST |
| 2047 |           || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) { |
| 2048 |            break; |
| 2049 |          } |
| 2050 |          ADD_OP1_TRACE_GUARD(); |
| 2051 |          break; |
| 2052 |        case ZEND_INIT_DYNAMIC_CALL: |
| 2053 |          if (orig_op2_type == IS_OBJECT && op2_ce == zend_ce_closure) { |
| 2054 |            ADD_OP2_TRACE_GUARD(); |
| 2055 |          } |
| 2056 |          break; |
| 2057 |        case ZEND_SEND_ARRAY: |
| 2058 |        case ZEND_SEND_UNPACK: |
| 2059 |        case ZEND_CHECK_UNDEF_ARGS: |
| 2060 |        case ZEND_INCLUDE_OR_EVAL: |
| 2061 |          max_used_stack = used_stack = -1; |
| 2062 |          break; |
| 2063 |        case ZEND_TYPE_CHECK: |
| 2064 |          if (opline->extended_value == MAY_BE_RESOURCE) { |
| 2065 |            // TODO: support for is_resource() ??? |
| 2066 |            break; |
| 2067 |          } |
| 2068 |          if (op1_type != IS_UNKNOWN |
| 2069 |           && (opline->extended_value == (1 << op1_type) |
| 2070 |           || )) { |
| 2071 |            /* add guards only for exact checks, to avoid code duplication */ |
| 2072 |            ADD_OP1_TRACE_GUARD(); |
| 2073 |          } |
| 2074 |          break; |
| 2075 |        case ZEND_ROPE_INIT: |
| 2076 |        case ZEND_ROPE_ADD: |
| 2077 |        case ZEND_ROPE_END: |
| 2078 |          if (op2_type == IS_STRING) { |
| 2079 |            ADD_OP2_TRACE_GUARD(); |
| 2080 |          } |
| 2081 |          break; |
| 2082 |        default: |
| 2083 |          break; |
| 2084 |      } |
| 2085 |      len = zend_jit_trace_op_len(opline); |
| 2086 |      if (ssa->var_info) { |
| 2087 |        /* Add statically inferred ranges */ |
| 2088 |        if (ssa_ops[idx].op1_def >= 0) { |
| 2089 |          zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def); |
| 2090 |        } |
| 2091 |        if (ssa_ops[idx].op2_def >= 0) { |
| 2092 |          zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def); |
| 2093 |        } |
| 2094 |        if (ssa_ops[idx].result_def >= 0) { |
| 2095 |          zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def); |
| 2096 |        } |
| 2097 |        if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) { |
| 2098 |          if (ssa_ops[idx+1].op1_def >= 0) { |
| 2099 |            zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def); |
| 2100 |          } |
| 2101 |          if (ssa_ops[idx+1].op2_def >= 0) { |
| 2102 |            zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def); |
| 2103 |          } |
| 2104 |          if (ssa_ops[idx+1].result_def >= 0) { |
| 2105 |            zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].result_def); |
| 2106 |          } |
| 2107 |        } |
| 2108 |      } else { |
| 2109 |        if (ssa_ops[idx].op1_def >= 0) { |
| 2110 |          ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var)); |
| 2111 |          if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) { |
| 2112 |            zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def); |
| 2113 |          } |
| 2114 |        } |
| 2115 |        if (ssa_ops[idx].op2_def >= 0) { |
| 2116 |          ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var)); |
| 2117 |          if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) { |
| 2118 |            zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def); |
| 2119 |          } |
| 2120 |        } |
| 2121 |        if (ssa_ops[idx].result_def >= 0) { |
| 2122 |          ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var)); |
| 2123 |          if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) { |
| 2124 |            zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def); |
| 2125 |          } |
| 2126 |        } |
| 2127 |        if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) { |
| 2128 |          if (ssa_ops[idx+1].op1_def >= 0) { |
| 2129 |            ssa_vars[ssa_ops[idx+1].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op1.var)); |
| 2130 |            if (ssa_ops[idx+1].op1_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op1_use].type & MAY_BE_REF)) { |
| 2131 |              zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def); |
| 2132 |            } |
| 2133 |          } |
| 2134 |          if (ssa_ops[idx+1].op2_def >= 0) { |
| 2135 |            ssa_vars[ssa_ops[idx+1].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op2.var)); |
| 2136 |            if (ssa_ops[idx+1].op2_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op2_use].type & MAY_BE_REF)) { |
| 2137 |              zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def); |
| 2138 |            } |
| 2139 |          } |
| 2140 |          if (ssa_ops[idx+1].result_def >= 0) { |
| 2141 |            ssa_vars[ssa_ops[idx+1].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->result.var)); |
| 2142 |            if (ssa_ops[idx+1].result_use < 0 || !(ssa_var_info[ssa_ops[idx+1].result_use].type & MAY_BE_REF)) { |
| 2143 |              zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].result_def); |
| 2144 |            } |
| 2145 |          } |
| 2146 |        } |
| 2147 |      } |
| 2148 |      if (opline->opcode == ZEND_RECV_INIT |
| 2149 |       && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { |
| 2150 |        /* RECV_INIT always copy the constant */ |
| 2151 |        ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2)); |
| 2152 |      } else if ((opline->opcode == ZEND_FE_FETCH_R || opline->opcode == ZEND_FE_FETCH_RW) |
| 2153 |       && ssa_opcodes[idx + 1] == ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) { |
| 2154 |        if (ssa_ops[idx].op2_use >= 0 && ssa_ops[idx].op2_def >= 0) { |
| 2155 |          ssa_var_info[ssa_ops[idx].op2_def] = ssa_var_info[ssa_ops[idx].op2_use]; |
| 2156 |        } |
| 2157 |      } else { |
| 2158 |        if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) { |
| 2159 |          // TODO: |
| 2160 |          assert(0); |
| 2161 |        } |
| 2162 |        if (opline->opcode == ZEND_ASSIGN_DIM_OP |
| 2163 |         && ssa_ops[idx].op1_def > 0 |
| 2164 |         && op1_type == IS_ARRAY |
| 2165 |         && (orig_op1_type & IS_TRACE_PACKED) |
| 2166 |         && val_type != IS_UNKNOWN |
| 2167 |         && val_type != IS_UNDEF |
| 2168 |         && ((opline->op2_type == IS_CONST |
| 2169 |         && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG) |
| 2170 |         || (opline->op2_type != IS_CONST |
| 2171 |         && op2_type == IS_LONG))) { |
| 2172 |          zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op1_def]; |
| 2173 | |
| 2174 |          info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH); |
| 2175 |        } |
| 2176 |      } |
| 2177 |      if (ssa->var_info) { |
| 2178 |        /* Add statically inferred restrictions */ |
| 2179 |        if (ssa_ops[idx].op1_def >= 0) { |
| 2180 |          if (opline->opcode == ZEND_SEND_VAR_EX |
| 2181 |           && frame |
| 2182 |           && frame->call |
| 2183 |           && frame->call->func |
| 2184 |           && !ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) { |
| 2185 |            ssa_var_info[ssa_ops[idx].op1_def] = ssa_var_info[ssa_ops[idx].op1_use]; |
| 2186 |            ssa_var_info[ssa_ops[idx].op1_def].type &= ~MAY_BE_GUARD; |
| 2187 |            if (ssa_var_info[ssa_ops[idx].op1_def].type & MAY_BE_RC1) { |
| 2188 |              ssa_var_info[ssa_ops[idx].op1_def].type |= MAY_BE_RCN; |
| 2189 |            } |
| 2190 |          } else { |
| 2191 |            zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def); |
| 2192 |          } |
| 2193 |        } |
| 2194 |        if (ssa_ops[idx].op2_def >= 0) { |
| 2195 |          if ((opline->opcode != ZEND_FE_FETCH_R && opline->opcode != ZEND_FE_FETCH_RW) |
| 2196 |           || ssa_opcodes[idx + 1] != ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) { |
| 2197 |            zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def); |
| 2198 |          } |
| 2199 |        } |
| 2200 |        if (ssa_ops[idx].result_def >= 0) { |
| 2201 |          zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def); |
| 2202 |        } |
| 2203 |      } |
| 2204 |      idx++; |
| 2205 |      while (len > 1) { |
| 2206 |        opline++; |
| 2207 |        if (opline->opcode != ZEND_OP_DATA) { |
| 2208 |          if (ssa->var_info) { |
| 2209 |            /* Add statically inferred ranges */ |
| 2210 |            if (ssa_ops[idx].op1_def >= 0) { |
| 2211 |              zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def); |
| 2212 |            } |
| 2213 |            if (ssa_ops[idx].op2_def >= 0) { |
| 2214 |              zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def); |
| 2215 |            } |
| 2216 |            if (ssa_ops[idx].result_def >= 0) { |
| 2217 |              zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def); |
| 2218 |            } |
| 2219 |          } else { |
| 2220 |            if (ssa_ops[idx].op1_def >= 0) { |
| 2221 |              ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var)); |
| 2222 |              if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) { |
| 2223 |                zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def); |
| 2224 |              } |
| 2225 |            } |
| 2226 |            if (ssa_ops[idx].op2_def >= 0) { |
| 2227 |              ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var)); |
| 2228 |              if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) { |
| 2229 |                zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def); |
| 2230 |              } |
| 2231 |            } |
| 2232 |            if (ssa_ops[idx].result_def >= 0) { |
| 2233 |              ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var)); |
| 2234 |              if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) { |
| 2235 |                zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def); |
| 2236 |              } |
| 2237 |            } |
| 2238 |          } |
| 2239 |          if (opline->opcode == ZEND_RECV_INIT |
| 2240 |           && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { |
| 2241 |            /* RECV_INIT always copy the constant */ |
| 2242 |            ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2)); |
| 2243 |          } else { |
| 2244 |            if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) { |
| 2245 |              // TODO: |
| 2246 |              assert(0); |
| 2247 |            } |
| 2248 |          } |
| 2249 |        } |
| 2250 |        if (ssa->var_info) { |
| 2251 |          /* Add statically inferred restrictions */ |
| 2252 |          if (ssa_ops[idx].op1_def >= 0) { |
| 2253 |            zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def); |
| 2254 |          } |
| 2255 |          if (ssa_ops[idx].op2_def >= 0) { |
| 2256 |            zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def); |
| 2257 |          } |
| 2258 |          if (ssa_ops[idx].result_def >= 0) { |
| 2259 |            zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def); |
| 2260 |          } |
| 2261 |        } |
| 2262 |        idx++; |
| 2263 |        len--; |
| 2264 |      } |
| 2265 | |
| 2266 |    } else if (p->op == ZEND_JIT_TRACE_ENTER) { |
| 2267 |      op_array = p->op_array; |
| 2268 |      jit_extension = |
| 2269 |        (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 2270 |      ssa = &jit_extension->func_info.ssa; |
| 2271 | |
| 2272 |      call = frame->call; |
| 2273 |      if (!call) { |
| 2274 |        /* Trace missed INIT_FCALL opcode */ |
| 2275 |        call = top; |
| 2276 |        TRACE_FRAME_INIT(call, op_array, 0, 0); |
| 2277 |        call->used_stack = 0; |
| 2278 |        top = zend_jit_trace_call_frame(top, op_array); |
| 2279 |        for (i = 0; i < op_array->last_var + op_array->T; i++) { |
| 2280 |          SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); |
| 2281 |        } |
| 2282 |      } else { |
| 2283 |        ZEND_ASSERT(&call->func->op_array == op_array); |
| 2284 |      } |
| 2285 |      frame->call = call->prev; |
| 2286 |      call->prev = frame; |
| 2287 |      TRACE_FRAME_SET_RETURN_SSA_VAR(call, find_return_ssa_var(p - 1, ssa_ops + (idx - 1))); |
| 2288 |      frame = call; |
| 2289 | |
| 2290 |      level++; |
| 2291 |      i = 0; |
| 2292 |      v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); |
| 2293 |      while (i < op_array->last_var) { |
| 2294 |        ssa_vars[v].var = i; |
| 2295 |        if (i < op_array->num_args) { |
| 2296 |          if (ssa->var_info |
| 2297 |           && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) { |
| 2298 |            /* pass */ |
| 2299 |          } else { |
| 2300 |            ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i); |
| 2301 |            if (op_array->arg_info) { |
| 2302 |              zend_arg_info *arg_info = &op_array->arg_info[i]; |
| 2303 |              zend_class_entry *ce; |
| 2304 |              uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce); |
| 2305 | |
| 2306 |              if (ZEND_ARG_SEND_MODE(arg_info)) { |
| 2307 |                tmp |= MAY_BE_REF; |
| 2308 |              } |
| 2309 |              ssa_var_info[v].type = tmp; |
| 2310 |              ssa_var_info[v].ce = ce; |
| 2311 |              ssa_var_info[v].is_instanceof = 1; |
| 2312 |            } else { |
| 2313 |              ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; |
| 2314 |            } |
| 2315 |          } |
| 2316 |        } else { |
| 2317 |          if (ssa->vars) { |
| 2318 |            ssa_vars[v].no_val = ssa->vars[i].no_val; |
| 2319 |            ssa_vars[v].alias = ssa->vars[i].alias; |
| 2320 |          } else { |
| 2321 |            ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i); |
| 2322 |          } |
| 2323 |          if (ssa_vars[v].alias == NO_ALIAS) { |
| 2324 |            ssa_var_info[v].type = MAY_BE_UNDEF; |
| 2325 |          } else { |
| 2326 |            ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; |
| 2327 |          } |
| 2328 |        } |
| 2329 |        if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) |
| 2330 |         && i < op_array->num_args) { |
| 2331 |          /* Propagate argument type */ |
| 2332 |          ssa_var_info[v].type &= STACK_INFO(frame->stack, i); |
| 2333 |        } |
| 2334 |        i++; |
| 2335 |        v++; |
| 2336 |      } |
| 2337 |    } else if (p->op == ZEND_JIT_TRACE_BACK) { |
| 2338 |      op_array = p->op_array; |
| 2339 |      jit_extension = |
| 2340 |        (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 2341 |      ssa = &jit_extension->func_info.ssa; |
| 2342 |      if (level == 0) { |
| 2343 |        i = 0; |
| 2344 |        v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); |
| 2345 |        while (i < op_array->last_var) { |
| 2346 |          ssa_vars[v].var = i; |
| 2347 |          if (!ssa->var_info |
| 2348 |           || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) { |
| 2349 |            ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; |
| 2350 |          } |
| 2351 |          i++; |
| 2352 |          v++; |
| 2353 |        } |
| 2354 |        while (i < op_array->last_var + op_array->T) { |
| 2355 |          ssa_vars[v].var = i; |
| 2356 |          if (!ssa->var_info |
| 2357 |           || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) { |
| 2358 |            ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; |
| 2359 |          } |
| 2360 |          i++; |
| 2361 |          v++; |
| 2362 |        } |
| 2363 |        if (return_value_info.type != 0) { |
| 2364 |          zend_jit_trace_rec *q = p + 1; |
| 2365 |          while (q->op == ZEND_JIT_TRACE_INIT_CALL) { |
| 2366 |            q++; |
| 2367 |          } |
| 2368 |          if (q->op == ZEND_JIT_TRACE_VM |
| 2369 |           || (q->op == ZEND_JIT_TRACE_END |
| 2370 |           && q->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)) { |
| 2371 |            const zend_op *opline = q->opline - 1; |
| 2372 |            if (opline->result_type != IS_UNUSED) { |
| 2373 |              ssa_var_info[ |
| 2374 |                ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info) + |
| 2375 |                EX_VAR_TO_NUM(opline->result.var)] = return_value_info; |
| 2376 |            } |
| 2377 |          } |
| 2378 |          memset(&return_value_info, 0, sizeof(return_value_info)); |
| 2379 |        } |
| 2380 |      } else { |
| 2381 |        level--; |
| 2382 |        if (return_value_info.type != 0) { |
| 2383 |          if ((p+1)->op == ZEND_JIT_TRACE_VM) { |
| 2384 |            const zend_op *opline = (p+1)->opline - 1; |
| 2385 |            if (opline->result_type != IS_UNUSED) { |
| 2386 |              if (TRACE_FRAME_RETURN_SSA_VAR(frame) >= 0) { |
| 2387 |                ssa_var_info[TRACE_FRAME_RETURN_SSA_VAR(frame)] = return_value_info; |
| 2388 |              } |
| 2389 |            } |
| 2390 |          } |
| 2391 |          memset(&return_value_info, 0, sizeof(return_value_info)); |
| 2392 |        } |
| 2393 |      } |
| 2394 | |
| 2395 |      top = frame; |
| 2396 |      if (frame->prev) { |
| 2397 |        if (used_stack > 0) { |
| 2398 |          used_stack -= frame->used_stack; |
| 2399 |        } |
| 2400 |        frame = frame->prev; |
| 2401 |        ZEND_ASSERT(&frame->func->op_array == op_array); |
| 2402 |      } else { |
| 2403 |        max_used_stack = used_stack = -1; |
| 2404 |        frame = zend_jit_trace_ret_frame(frame, op_array); |
| 2405 |        TRACE_FRAME_INIT(frame, op_array, 0, 0); |
| 2406 |        TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1); |
| 2407 |        frame->used_stack = 0; |
| 2408 |        for (i = 0; i < op_array->last_var + op_array->T; i++) { |
| 2409 |          SET_STACK_TYPE(frame->stack, i, IS_UNKNOWN, 1); |
| 2410 |        } |
| 2411 |      } |
| 2412 | |
| 2413 |    } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { |
| 2414 |      call = top; |
| 2415 |      TRACE_FRAME_INIT(call, p->func, 0, 0); |
| 2416 |      call->prev = frame->call; |
| 2417 |      call->used_stack = 0; |
| 2418 |      frame->call = call; |
| 2419 |      top = zend_jit_trace_call_frame(top, p->op_array); |
| 2420 |      if (p->func && p->func->type == ZEND_USER_FUNCTION) { |
| 2421 |        for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) { |
| 2422 |          SET_STACK_INFO(call->stack, i, -1); |
| 2423 |        } |
| 2424 |      } |
| 2425 |      if (used_stack >= 0 |
| 2426 |       && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { |
| 2427 |        if (p->func == NULL || (p-1)->op != ZEND_JIT_TRACE_VM) { |
| 2428 |          max_used_stack = used_stack = -1; |
| 2429 |        } else { |
| 2430 |          const zend_op *opline = (p-1)->opline; |
| 2431 | |
| 2432 |          switch (opline->opcode) { |
| 2433 |            case ZEND_INIT_FCALL: |
| 2434 |            case ZEND_INIT_FCALL_BY_NAME: |
| 2435 |            case ZEND_INIT_NS_FCALL_BY_NAME: |
| 2436 |            case ZEND_INIT_METHOD_CALL: |
| 2437 |            case ZEND_INIT_DYNAMIC_CALL: |
| 2438 |            //case ZEND_INIT_STATIC_METHOD_CALL: |
| 2439 |            //case ZEND_INIT_USER_CALL: |
| 2440 |            //case ZEND_NEW: |
| 2441 |              frame->used_stack = zend_vm_calc_used_stack(opline->extended_value, (zend_function*)p->func); |
| 2442 |              used_stack += frame->used_stack; |
| 2443 |              if (used_stack > max_used_stack) { |
| 2444 |                max_used_stack = used_stack; |
| 2445 |              } |
| 2446 |              break; |
| 2447 |            default: |
| 2448 |              max_used_stack = used_stack = -1; |
| 2449 |          } |
| 2450 |        } |
| 2451 |      } |
| 2452 |    } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { |
| 2453 |      call = frame->call; |
| 2454 |      if (call) { |
| 2455 |        top = call; |
| 2456 |        frame->call = call->prev; |
| 2457 |      } |
| 2458 | |
| 2459 |      if (idx > 0 |
| 2460 |       && ssa_ops[idx-1].result_def >= 0 |
| 2461 |       && (p->func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE) |
| 2462 |       && !(p->func->common.fn_flags & ZEND_ACC_RETURN_REFERENCE)) { |
| 2463 |        ZEND_ASSERT(ssa_opcodes[idx-1] == opline); |
| 2464 |        ZEND_ASSERT(opline->opcode == ZEND_DO_ICALL || |
| 2465 |          opline->opcode == ZEND_DO_FCALL || |
| 2466 |          opline->opcode == ZEND_DO_FCALL_BY_NAME); |
| 2467 | |
| 2468 |        if (opline->result_type != IS_UNDEF) { |
| 2469 |          zend_class_entry *ce; |
| 2470 |          const zend_function *func = p->func; |
| 2471 |          zend_arg_info *ret_info = func->common.arg_info - 1; |
| 2472 |          uint32_t ret_type = zend_fetch_arg_info_type(NULL, ret_info, &ce); |
| 2473 | |
| 2474 |          ssa_var_info[ssa_ops[idx-1].result_def].type &= ret_type; |
| 2475 |        } |
| 2476 |      } |
| 2477 |    } else if (p->op == ZEND_JIT_TRACE_END) { |
| 2478 |      break; |
| 2479 |    } |
| 2480 |  } |
| 2481 | |
| 2482 |  ((zend_tssa*)tssa)->used_stack = max_used_stack; |
| 2483 | |
| 2484 |  if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP |
| 2485 |   || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL |
| 2486 |   || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 2487 |    /* Propagate guards through Phi sources */ |
| 2488 |    zend_ssa_phi *phi = tssa->blocks[1].phis; |
| 2489 | |
| 2490 |    op_array = trace_buffer->op_array; |
| 2491 |    jit_extension = |
| 2492 |      (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 2493 |    ssa = &jit_extension->func_info.ssa; |
| 2494 | |
| 2495 |    while (phi) { |
| 2496 |      uint32_t t = ssa_var_info[phi->ssa_var].type; |
| 2497 | |
| 2498 |      if ((t & MAY_BE_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) { |
| 2499 |        uint32_t t0 = ssa_var_info[phi->sources[0]].type; |
| 2500 |        uint32_t t1 = ssa_var_info[phi->sources[1]].type; |
| 2501 | |
| 2502 |        if ( ) { |
| 2503 |          if (!((t0 | t1) & MAY_BE_GUARD)) { |
| 2504 |            ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD; |
| 2505 |          } |
| 2506 |        } else if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) { |
| 2507 |          if (!(t1 & MAY_BE_GUARD) |
| 2508 |           || is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) { |
| 2509 |            ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD; |
| 2510 |            t0 = (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) | |
| 2511 |              (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) | |
| 2512 |              MAY_BE_GUARD; |
| 2513 |            if (!(t0 & MAY_BE_ARRAY)) { |
| 2514 |              t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY); |
| 2515 |            } |
| 2516 |            if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { |
| 2517 |              t0 &= ~(MAY_BE_RC1|MAY_BE_RCN); |
| 2518 |            } |
| 2519 |            ssa_var_info[phi->sources[0]].type = t0; |
| 2520 |            ssa_var_info[phi->sources[0]].type = t0; |
| 2521 |          } |
| 2522 |        } else { |
| 2523 |          if ((t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) { |
| 2524 |            t0 = (t & t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) | |
| 2525 |              (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) | |
| 2526 |              MAY_BE_GUARD; |
| 2527 |            if (!(t0 & MAY_BE_ARRAY)) { |
| 2528 |              t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY); |
| 2529 |            } |
| 2530 |            if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { |
| 2531 |              t0 &= ~(MAY_BE_RC1|MAY_BE_RCN); |
| 2532 |            } |
| 2533 |            ssa_var_info[phi->sources[0]].type = t0; |
| 2534 |          } |
| 2535 |          if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) { |
| 2536 |            if (((t & t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != 0 |
| 2537 |             && is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) { |
| 2538 |              t1 = (t & t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) | |
| 2539 |                (t1 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) | |
| 2540 |                MAY_BE_GUARD; |
| 2541 |              if (!(t1 & MAY_BE_ARRAY)) { |
| 2542 |                t1 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY); |
| 2543 |              } |
| 2544 |              if (!(t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { |
| 2545 |                t1 &= ~(MAY_BE_RC1|MAY_BE_RCN); |
| 2546 |              } |
| 2547 |              ssa_var_info[phi->sources[1]].type = t1; |
| 2548 |              ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD; |
| 2549 |            } |
| 2550 |          } |
| 2551 |        } |
| 2552 |        t = ssa_var_info[phi->ssa_var].type; |
| 2553 |      } |
| 2554 | |
| 2555 |      if ((t & MAY_BE_PACKED_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) { |
| 2556 |        uint32_t t0 = ssa_var_info[phi->sources[0]].type; |
| 2557 |        uint32_t t1 = ssa_var_info[phi->sources[1]].type; |
| 2558 | |
| 2559 |        if ( ) { |
| 2560 |          if (!((t0 | t1) & MAY_BE_PACKED_GUARD)) { |
| 2561 |            ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD; |
| 2562 |          } |
| 2563 |        } else if ((t1 & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) { |
| 2564 |          if (!(t1 & MAY_BE_PACKED_GUARD)) { |
| 2565 |            ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD; |
| 2566 |            ssa_var_info[phi->sources[0]].type = |
| 2567 |              (t0 & ~MAY_BE_ARRAY_KEY_ANY) | (t & MAY_BE_ARRAY_KEY_ANY) | MAY_BE_PACKED_GUARD; |
| 2568 |          } |
| 2569 |        } |
| 2570 |      } |
| 2571 |      phi = phi->next; |
| 2572 |    } |
| 2573 |  } |
| 2574 | |
| 2575 |  if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_TSSA)) { |
| 2576 |    if (parent_trace) { |
| 2577 |      fprintf(stderr, "---- TRACE %d TSSA start (side trace %d/%d) %s%s%s() %s:%d\n", |
| 2578 |        ZEND_JIT_TRACE_NUM, |
| 2579 |        parent_trace, |
| 2580 |        exit_num, |
| 2581 |        trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "", |
| 2582 |        trace_buffer->op_array->scope ? "::" : "", |
| 2583 |        trace_buffer->op_array->function_name ? |
| 2584 |          ZSTR_VAL(trace_buffer->op_array->function_name) : "$main", |
| 2585 |        ZSTR_VAL(trace_buffer->op_array->filename), |
| 2586 |        trace_buffer[1].opline->lineno); |
| 2587 |    } else { |
| 2588 |      fprintf(stderr, "---- TRACE %d TSSA start (%s) %s%s%s() %s:%d\n", |
| 2589 |        ZEND_JIT_TRACE_NUM, |
| 2590 |        zend_jit_trace_star_desc(trace_buffer->start), |
| 2591 |        trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "", |
| 2592 |        trace_buffer->op_array->scope ? "::" : "", |
| 2593 |        trace_buffer->op_array->function_name ? |
| 2594 |          ZSTR_VAL(trace_buffer->op_array->function_name) : "$main", |
| 2595 |        ZSTR_VAL(trace_buffer->op_array->filename), |
| 2596 |        trace_buffer[1].opline->lineno); |
| 2597 |    } |
| 2598 |    zend_jit_dump_trace(trace_buffer, tssa); |
| 2599 |    if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LINK) { |
| 2600 |      uint32_t idx = trace_buffer[1].last; |
| 2601 |      uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler); |
| 2602 |      fprintf(stderr, "---- TRACE %d TSSA stop (link to %d)\n", |
| 2603 |        ZEND_JIT_TRACE_NUM, |
| 2604 |        link_to); |
| 2605 |    } else { |
| 2606 |      fprintf(stderr, "---- TRACE %d TSSA stop (%s)\n", |
| 2607 |        ZEND_JIT_TRACE_NUM, |
| 2608 |        zend_jit_trace_stop_description[trace_buffer->stop]); |
| 2609 |    } |
| 2610 |  } |
| 2611 | |
| 2612 |  return tssa; |
| 2613 | } |
| 2614 | |
| 2615 | static void zend_jit_close_var(zend_jit_trace_stack *stack, uint32_t n, int *start, int *end, uint8_t *flags, int line) |
| 2616 | { |
| 2617 |  int32_t var = STACK_VAR(stack, n); |
| 2618 | |
| 2619 |  if (var >= 0 && start[var] >= 0 && !(flags[var] & ZREG_LAST_USE)) { |
| 2620 |    // TODO: shrink interval to last side exit ???? |
| 2621 |    end[var] = line; |
| 2622 |  } |
| 2623 | } |
| 2624 | |
| 2625 | static void zend_jit_trace_use_var(int line, int var, int def, int use_chain, int *start, int *end, uint8_t *flags, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array, const zend_ssa *op_array_ssa) |
| 2626 | { |
| 2627 |  ZEND_ASSERT(start[var] >= 0); |
| 2628 |  ZEND_ASSERT(!(flags[var] & ZREG_LAST_USE)); |
| 2629 |  end[var] = line; |
| 2630 |  if (def >= 0) { |
| 2631 |    flags[var] |= ZREG_LAST_USE; |
| 2632 |  } else if ( ) { |
| 2633 |    flags[var] |= ZREG_LAST_USE; |
| 2634 |  } else if (use_chain >= 0 && !zend_ssa_is_no_val_use(ssa_opcodes[use_chain], ssa->ops + use_chain, var)) { |
| 2635 |    /* pass */ |
| 2636 |  } else if (op_array_ssa->vars) { |
| 2637 |    uint32_t use = ssa_opcodes[line] - op_array->opcodes; |
| 2638 | |
| 2639 |    if (ssa->ops[line].op1_use == var) { |
| 2640 |      if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op1_use, use)) { |
| 2641 |        flags[var] |= ZREG_LAST_USE; |
| 2642 |      } |
| 2643 |    } else if (ssa->ops[line].op2_use == var) { |
| 2644 |      if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op2_use, use)) { |
| 2645 |        flags[var] |= ZREG_LAST_USE; |
| 2646 |      } |
| 2647 |    } else if (ssa->ops[line].result_use == var) { |
| 2648 |      if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].result_use, use)) { |
| 2649 |        flags[var] |= ZREG_LAST_USE; |
| 2650 |      } |
| 2651 |    } |
| 2652 |  } |
| 2653 | } |
| 2654 | |
| 2655 | static zend_lifetime_interval** zend_jit_trace_allocate_registers(zend_jit_trace_rec *trace_buffer, zend_ssa *ssa, uint32_t parent_trace, uint32_t exit_num) |
| 2656 | { |
| 2657 |  const zend_op **ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes; |
| 2658 |  zend_jit_trace_rec *p; |
| 2659 |  const zend_op_array *op_array; |
| 2660 |  zend_jit_op_array_trace_extension *jit_extension; |
| 2661 |  const zend_ssa *op_array_ssa; |
| 2662 |  const zend_ssa_op *ssa_op; |
| 2663 |  int i, j, idx, count, level; |
| 2664 |  int *start, *end; |
| 2665 |  uint8_t *flags; |
| 2666 |  const zend_op_array **vars_op_array; |
| 2667 |  zend_lifetime_interval **intervals, *list, *ival; |
| 2668 |  void *checkpoint; |
| 2669 |  zend_jit_trace_stack_frame *frame; |
| 2670 |  zend_jit_trace_stack *stack; |
| 2671 |  uint32_t parent_vars_count = parent_trace ? |
| 2672 |    zend_jit_traces[parent_trace].exit_info[exit_num].stack_size : 0; |
| 2673 |  zend_jit_trace_stack *parent_stack = parent_trace ? |
| 2674 |    zend_jit_traces[parent_trace].stack_map + |
| 2675 |    zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset : NULL; |
| 2676 |  ALLOCA_FLAG(use_heap); |
| 2677 | |
| 2678 |  ZEND_ASSERT(ssa->var_info != NULL); |
| 2679 | |
| 2680 |  start = do_alloca(sizeof(int) * ssa->vars_count * 2 + |
| 2681 |    ZEND_MM_ALIGNED_SIZE(sizeof(uint8_t) * ssa->vars_count) + |
| 2682 |    ZEND_MM_ALIGNED_SIZE(sizeof(zend_op_array*) * ssa->vars_count), |
| 2683 |    use_heap); |
| 2684 |  if (!start) { |
| 2685 |    return NULL; |
| 2686 |  } |
| 2687 |  end = start + ssa->vars_count; |
| 2688 |  flags = (uint8_t*)(end + ssa->vars_count); |
| 2689 |  vars_op_array = (const zend_op_array**)(flags + ZEND_MM_ALIGNED_SIZE(sizeof(uint8_t) * ssa->vars_count)); |
| 2690 | |
| 2691 |  memset(start, -1, sizeof(int) * ssa->vars_count * 2); |
| 2692 |  memset(flags, 0, sizeof(uint8_t) * ssa->vars_count); |
| 2693 |  memset(ZEND_VOIDP(vars_op_array), 0, sizeof(zend_op_array*) * ssa->vars_count); |
| 2694 | |
| 2695 |  op_array = trace_buffer->op_array; |
| 2696 |  jit_extension = |
| 2697 |    (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 2698 |  op_array_ssa = &jit_extension->func_info.ssa; |
| 2699 |  frame = JIT_G(current_frame); |
| 2700 |  frame->prev = NULL; |
| 2701 |  frame->func = (const zend_function*)op_array; |
| 2702 |  stack = frame->stack; |
| 2703 | |
| 2704 |  count = 0; |
| 2705 | |
| 2706 |  i = 0; |
| 2707 |  j = op_array->last_var; |
| 2708 |  if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) { |
| 2709 |    j += op_array->T; |
| 2710 |  } |
| 2711 |  while (i < j) { |
| 2712 |    SET_STACK_VAR(stack, i, i); |
| 2713 |    vars_op_array[i] = op_array; |
| 2714 |    /* We don't start intervals for variables used in Phi */ |
| 2715 |    if ((ssa->vars[i].use_chain >= 0 /*|| ssa->vars[i].phi_use_chain*/) |
| 2716 |     && !zend_ssa_is_no_val_use(ssa_opcodes[ssa->vars[i].use_chain], ssa->ops + ssa->vars[i].use_chain, i) |
| 2717 |     && ssa->vars[i].alias == NO_ALIAS |
| 2718 |     && zend_jit_var_supports_reg(ssa, i)) { |
| 2719 |      start[i] = 0; |
| 2720 |      if ( |
| 2721 |       |
| 2722 |       ) { |
| 2723 |        /* We will try to reuse register from parent trace */ |
| 2724 |        flags[i] = STACK_FLAGS(parent_stack, i); |
| 2725 |        count += 2; |
| 2726 |      } else { |
| 2727 |        flags[i] = ZREG_LOAD; |
| 2728 |        count++; |
| 2729 |      } |
| 2730 |    } |
| 2731 |    i++; |
| 2732 |  } |
| 2733 | |
| 2734 |  if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { |
| 2735 |    j = op_array->last_var + op_array->T; |
| 2736 |    while (i < j) { |
| 2737 |      SET_STACK_VAR(stack, i, -1); |
| 2738 |      i++; |
| 2739 |    } |
| 2740 |  } |
| 2741 | |
| 2742 |  if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP |
| 2743 |   || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL |
| 2744 |   || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 2745 |    zend_ssa_phi *phi = ssa->blocks[1].phis; |
| 2746 | |
| 2747 |    while (phi) { |
| 2748 |      SET_STACK_VAR(stack, phi->var, phi->ssa_var); |
| 2749 |      vars_op_array[phi->ssa_var] = op_array; |
| 2750 |      if (ssa->vars[phi->ssa_var].use_chain >= 0 |
| 2751 |       && ssa->vars[phi->ssa_var].alias == NO_ALIAS |
| 2752 |       && zend_jit_var_supports_reg(ssa, phi->ssa_var)) { |
| 2753 |        start[phi->ssa_var] = 0; |
| 2754 |        count++; |
| 2755 |      } |
| 2756 |      phi = phi->next; |
| 2757 |    } |
| 2758 |  } |
| 2759 | |
| 2760 |  p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; |
| 2761 |  level = 0; |
| 2762 |  ssa_op = ssa->ops; |
| 2763 |  idx = 0; |
| 2764 |  for (;;p++) { |
| 2765 |    if (p->op == ZEND_JIT_TRACE_VM) { |
| 2766 |      const zend_op *opline = p->opline; |
| 2767 |      int len; |
| 2768 |      bool support_opline; |
| 2769 | |
| 2770 |      support_opline = |
| 2771 |        zend_jit_opline_supports_reg(op_array, ssa, opline, ssa_op, p); |
| 2772 | |
| 2773 |      if (support_opline |
| 2774 |       && opline->opcode == ZEND_ASSIGN |
| 2775 |       && opline->op1_type == IS_CV |
| 2776 |       && ssa_op->op1_def >= 0 |
| 2777 |       && ssa->vars[ssa_op->op1_def].alias != NO_ALIAS) { |
| 2778 |        /* avoid register allocation in case of possibility of indirect modification*/ |
| 2779 |        support_opline = 0; |
| 2780 |      } |
| 2781 | |
| 2782 |      if (ssa_op->op1_use >= 0 |
| 2783 |       && start[ssa_op->op1_use] >= 0 |
| 2784 |       && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) { |
| 2785 |        if (support_opline) { |
| 2786 |          zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa); |
| 2787 |          if (opline->op1_type != IS_CV) { |
| 2788 |            if (opline->opcode == ZEND_CASE |
| 2789 |             || opline->opcode == ZEND_CASE_STRICT |
| 2790 |             || opline->opcode == ZEND_SWITCH_LONG |
| 2791 |             || opline->opcode == ZEND_MATCH |
| 2792 |             || opline->opcode == ZEND_FETCH_LIST_R |
| 2793 |             || opline->opcode == ZEND_COPY_TMP |
| 2794 |             || opline->opcode == ZEND_SWITCH_STRING |
| 2795 |             || opline->opcode == ZEND_FE_FETCH_R |
| 2796 |             || opline->opcode == ZEND_FE_FETCH_RW |
| 2797 |             || opline->opcode == ZEND_FETCH_LIST_W |
| 2798 |             || opline->opcode == ZEND_VERIFY_RETURN_TYPE |
| 2799 |             || opline->opcode == ZEND_BIND_LEXICAL |
| 2800 |             || opline->opcode == ZEND_ROPE_ADD) { |
| 2801 |              /* The value is kept alive and may be used outside of the trace */ |
| 2802 |              flags[ssa_op->op1_use] |= ZREG_STORE; |
| 2803 |            } else { |
| 2804 |              flags[ssa_op->op1_use] |= ZREG_LAST_USE; |
| 2805 |            } |
| 2806 |          } |
| 2807 |        } else { |
| 2808 |          start[ssa_op->op1_use] = -1; |
| 2809 |          end[ssa_op->op1_use] = -1; |
| 2810 |          count--; |
| 2811 |        } |
| 2812 |      } |
| 2813 |      if (ssa_op->op2_use >= 0 |
| 2814 |       && ssa_op->op2_use != ssa_op->op1_use |
| 2815 |       && start[ssa_op->op2_use] >= 0 |
| 2816 |       && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op2_use)) { |
| 2817 |        if (support_opline) { |
| 2818 |          zend_jit_trace_use_var(idx, ssa_op->op2_use, ssa_op->op2_def, ssa_op->op2_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa); |
| 2819 |          if (opline->op2_type != IS_CV) { |
| 2820 |            flags[ssa_op->op2_use] |= ZREG_LAST_USE; |
| 2821 |          } |
| 2822 |        } else { |
| 2823 |          start[ssa_op->op2_use] = -1; |
| 2824 |          end[ssa_op->op2_use] = -1; |
| 2825 |          count--; |
| 2826 |        } |
| 2827 |      } |
| 2828 |      if (ssa_op->result_use >= 0 |
| 2829 |       && ssa_op->result_use != ssa_op->op1_use |
| 2830 |       && ssa_op->result_use != ssa_op->op2_use |
| 2831 |       && start[ssa_op->result_use] >= 0 |
| 2832 |       && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->result_use)) { |
| 2833 |        if (support_opline) { |
| 2834 |          zend_jit_trace_use_var(idx, ssa_op->result_use, ssa_op->result_def, ssa_op->res_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa); |
| 2835 |        } else { |
| 2836 |          start[ssa_op->result_use] = -1; |
| 2837 |          end[ssa_op->result_use] = -1; |
| 2838 |          count--; |
| 2839 |        } |
| 2840 |      } |
| 2841 | |
| 2842 |      if (ssa_op->op1_def >= 0) { |
| 2843 |        zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx); |
| 2844 |        SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def); |
| 2845 |      } |
| 2846 |      if (ssa_op->op2_def >= 0) { |
| 2847 |        zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op2.var), start, end, flags, idx); |
| 2848 |        SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op2.var), ssa_op->op2_def); |
| 2849 |      } |
| 2850 |      if (ssa_op->result_def >= 0) { |
| 2851 |        zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->result.var), start, end, flags, idx); |
| 2852 |        SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def); |
| 2853 |      } |
| 2854 | |
| 2855 |      if (support_opline) { |
| 2856 |        if (ssa_op->result_def >= 0 |
| 2857 |         && (ssa->vars[ssa_op->result_def].use_chain >= 0 |
| 2858 |       || ssa->vars[ssa_op->result_def].phi_use_chain) |
| 2859 |         && ssa->vars[ssa_op->result_def].alias == NO_ALIAS |
| 2860 |         && zend_jit_var_supports_reg(ssa, ssa_op->result_def)) { |
| 2861 |          if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD) |
| 2862 |           || opline->opcode == ZEND_PRE_INC |
| 2863 |           || opline->opcode == ZEND_PRE_DEC |
| 2864 |           || opline->opcode == ZEND_POST_INC |
| 2865 |           || opline->opcode == ZEND_POST_DEC |
| 2866 |           || opline->opcode == ZEND_ADD |
| 2867 |           || opline->opcode == ZEND_SUB |
| 2868 |           || opline->opcode == ZEND_MUL |
| 2869 |           || opline->opcode == ZEND_FETCH_DIM_R |
| 2870 |           || opline->opcode == ZEND_FETCH_CONSTANT) { |
| 2871 |            if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_DOUBLE) |
| 2872 |             || (opline->opcode != ZEND_PRE_INC && opline->opcode != ZEND_PRE_DEC)) { |
| 2873 |              start[ssa_op->result_def] = idx; |
| 2874 |              vars_op_array[ssa_op->result_def] = op_array; |
| 2875 |              count++; |
| 2876 |            } |
| 2877 |          } |
| 2878 |        } |
| 2879 |        if (ssa_op->op1_def >= 0 |
| 2880 |         && (ssa->vars[ssa_op->op1_def].use_chain >= 0 |
| 2881 |       || ssa->vars[ssa_op->op1_def].phi_use_chain) |
| 2882 |         && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS |
| 2883 |         && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)) { |
| 2884 |          start[ssa_op->op1_def] = idx; |
| 2885 |          vars_op_array[ssa_op->op1_def] = op_array; |
| 2886 |          count++; |
| 2887 |        } |
| 2888 |        if (ssa_op->op2_def >= 0 |
| 2889 |         && (ssa->vars[ssa_op->op2_def].use_chain >= 0 |
| 2890 |       || ssa->vars[ssa_op->op2_def].phi_use_chain) |
| 2891 |         && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS |
| 2892 |         && zend_jit_var_supports_reg(ssa, ssa_op->op2_def)) { |
| 2893 |          start[ssa_op->op2_def] = idx; |
| 2894 |          vars_op_array[ssa_op->op2_def] = op_array; |
| 2895 |          count++; |
| 2896 |        } |
| 2897 |      } |
| 2898 | |
| 2899 |      len = zend_jit_trace_op_len(opline); |
| 2900 |      switch (opline->opcode) { |
| 2901 |        case ZEND_ASSIGN_DIM: |
| 2902 |        case ZEND_ASSIGN_OBJ: |
| 2903 |        case ZEND_ASSIGN_STATIC_PROP: |
| 2904 |        case ZEND_ASSIGN_DIM_OP: |
| 2905 |        case ZEND_ASSIGN_OBJ_OP: |
| 2906 |        case ZEND_ASSIGN_STATIC_PROP_OP: |
| 2907 |        case ZEND_ASSIGN_OBJ_REF: |
| 2908 |        case ZEND_ASSIGN_STATIC_PROP_REF: |
| 2909 |          /* OP_DATA */ |
| 2910 |          ssa_op++; |
| 2911 |          opline++; |
| 2912 |          if (ssa_op->op1_use >= 0 |
| 2913 |           && start[ssa_op->op1_use] >= 0 |
| 2914 |           && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) { |
| 2915 |            if (support_opline) { |
| 2916 |              zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain, start, end, flags, ssa, ssa_opcodes, op_array, op_array_ssa); |
| 2917 |              if (opline->op1_type != IS_CV) { |
| 2918 |                flags[ssa_op->op1_use] |= ZREG_LAST_USE; |
| 2919 |              } |
| 2920 |            } else { |
| 2921 |              start[ssa_op->op1_use] = -1; |
| 2922 |              end[ssa_op->op1_use] = -1; |
| 2923 |              count--; |
| 2924 |            } |
| 2925 |          } |
| 2926 |          if (ssa_op->op1_def >= 0) { |
| 2927 |            zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx); |
| 2928 |            SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def); |
| 2929 |            if (support_opline |
| 2930 |             && (ssa->vars[ssa_op->op1_def].use_chain >= 0 |
| 2931 |           || ssa->vars[ssa_op->op1_def].phi_use_chain) |
| 2932 |             && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS |
| 2933 |             && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)) { |
| 2934 |              start[ssa_op->op1_def] = idx; |
| 2935 |              vars_op_array[ssa_op->op1_def] = op_array; |
| 2936 |              count++; |
| 2937 |            } |
| 2938 |          } |
| 2939 |          ssa_op++; |
| 2940 |          opline++; |
| 2941 |          idx+=2; |
| 2942 |          break; |
| 2943 |        case ZEND_RECV_INIT: |
| 2944 |         ssa_op++; |
| 2945 |          opline++; |
| 2946 |          idx++; |
| 2947 |          while (opline->opcode == ZEND_RECV_INIT) { |
| 2948 |            /* RECV_INIT doesn't support registers */ |
| 2949 |            if (ssa_op->result_def >= 0) { |
| 2950 |              zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->result.var), start, end, flags, idx); |
| 2951 |              SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def); |
| 2952 |            } |
| 2953 |            ssa_op++; |
| 2954 |            opline++; |
| 2955 |            idx++; |
| 2956 |          } |
| 2957 |          break; |
| 2958 |        case ZEND_BIND_GLOBAL: |
| 2959 |          ssa_op++; |
| 2960 |          opline++; |
| 2961 |          idx++; |
| 2962 |          while (opline->opcode == ZEND_BIND_GLOBAL) { |
| 2963 |            /* BIND_GLOBAL doesn't support registers */ |
| 2964 |            if (ssa_op->op1_def >= 0) { |
| 2965 |              zend_jit_close_var(stack, EX_VAR_TO_NUM(opline->op1.var), start, end, flags, idx); |
| 2966 |              SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def); |
| 2967 |            } |
| 2968 |            ssa_op++; |
| 2969 |            opline++; |
| 2970 |            idx++; |
| 2971 |          } |
| 2972 |          break; |
| 2973 |        default: |
| 2974 |          ssa_op += len; |
| 2975 |          idx += len; |
| 2976 |          break; |
| 2977 |      } |
| 2978 |    } else if (p->op == ZEND_JIT_TRACE_ENTER) { |
| 2979 |      /* New call frames */ |
| 2980 |      zend_jit_trace_stack_frame *prev_frame = frame; |
| 2981 | |
| 2982 |      frame = zend_jit_trace_call_frame(frame, op_array); |
| 2983 |      frame->prev = prev_frame; |
| 2984 |      frame->func = (const zend_function*)p->op_array; |
| 2985 |      stack = frame->stack; |
| 2986 |      op_array = p->op_array; |
| 2987 |      jit_extension = |
| 2988 |        (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 2989 |      op_array_ssa = &jit_extension->func_info.ssa; |
| 2990 |      j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); |
| 2991 |      for (i = 0; i < op_array->last_var; i++) { |
| 2992 |        SET_STACK_VAR(stack, i, j); |
| 2993 |        vars_op_array[j] = op_array; |
| 2994 |        if (ssa->vars[j].use_chain >= 0 |
| 2995 |         && ssa->vars[j].alias == NO_ALIAS |
| 2996 |         && zend_jit_var_supports_reg(ssa, j)) { |
| 2997 |          start[j] = idx; |
| 2998 |          flags[j] = ZREG_LOAD; |
| 2999 |          count++; |
| 3000 |        } |
| 3001 |        j++; |
| 3002 |      } |
| 3003 |      for (i = op_array->last_var; i < op_array->last_var + op_array->T; i++) { |
| 3004 |        SET_STACK_VAR(stack, i, -1); |
| 3005 |      } |
| 3006 |      level++; |
| 3007 |    } else if (p->op == ZEND_JIT_TRACE_BACK) { |
| 3008 |      /* Close exiting call frames */ |
| 3009 |      for (i = 0; i < op_array->last_var; i++) { |
| 3010 |        zend_jit_close_var(stack, i, start, end, flags, idx-1); |
| 3011 |      } |
| 3012 |      op_array = p->op_array; |
| 3013 |      jit_extension = |
| 3014 |        (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 3015 |      op_array_ssa = &jit_extension->func_info.ssa; |
| 3016 |      frame = zend_jit_trace_ret_frame(frame, op_array); |
| 3017 |      stack = frame->stack; |
| 3018 |      if (level == 0) { |
| 3019 |        /* New return frames */ |
| 3020 |        frame->prev = NULL; |
| 3021 |        frame->func = (const zend_function*)op_array; |
| 3022 |        j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); |
| 3023 |        for (i = 0; i < op_array->last_var + op_array->T; i++) { |
| 3024 |          SET_STACK_VAR(stack, i, j); |
| 3025 |          vars_op_array[j] = op_array; |
| 3026 |          if (ssa->vars[j].use_chain >= 0 |
| 3027 |           && ssa->vars[j].alias == NO_ALIAS |
| 3028 |           && zend_jit_var_supports_reg(ssa, j)) { |
| 3029 |            start[j] = idx; |
| 3030 |            flags[j] = ZREG_LOAD; |
| 3031 |            count++; |
| 3032 |          } |
| 3033 |          j++; |
| 3034 |        } |
| 3035 |      } else { |
| 3036 |        level--; |
| 3037 |      } |
| 3038 |    } else if (p->op == ZEND_JIT_TRACE_END) { |
| 3039 |      break; |
| 3040 |    } |
| 3041 |  } |
| 3042 | |
| 3043 |  if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP |
| 3044 |   || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL |
| 3045 |   || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 3046 |    zend_ssa_phi *phi = ssa->blocks[1].phis; |
| 3047 | |
| 3048 |    while (phi) { |
| 3049 |      i = phi->sources[1]; |
| 3050 |      if (start[i] >= 0 && !ssa->vars[phi->ssa_var].no_val) { |
| 3051 |        end[i] = idx; |
| 3052 |        flags[i] &= ~ZREG_LAST_USE; |
| 3053 |      } |
| 3054 |      phi = phi->next; |
| 3055 |    } |
| 3056 | |
| 3057 |    if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) { |
| 3058 |      for (i = 0; i < op_array->last_var; i++) { |
| 3059 |        if (start[i] >= 0 && !ssa->vars[i].phi_use_chain) { |
| 3060 |          end[i] = idx; |
| 3061 |          flags[i] &= ~ZREG_LAST_USE; |
| 3062 |        } else { |
| 3063 |          zend_jit_close_var(stack, i, start, end, flags, idx); |
| 3064 |        } |
| 3065 |      } |
| 3066 |    } |
| 3067 |  } else { |
| 3068 |    for (i = 0; i < op_array->last_var; i++) { |
| 3069 |      zend_jit_close_var(stack, i, start, end, flags, idx); |
| 3070 |    } |
| 3071 |    while (frame->prev) { |
| 3072 |      frame = frame->prev; |
| 3073 |      op_array = &frame->func->op_array; |
| 3074 |      stack = frame->stack; |
| 3075 |      for (i = 0; i < op_array->last_var; i++) { |
| 3076 |        zend_jit_close_var(stack, i, start, end, flags, idx); |
| 3077 |      } |
| 3078 |    } |
| 3079 |  } |
| 3080 | |
| 3081 |  if (!count) { |
| 3082 |    free_alloca(start, use_heap); |
| 3083 |    return NULL; |
| 3084 |  } |
| 3085 | |
| 3086 |  checkpoint = zend_arena_checkpoint(CG(arena)); |
| 3087 |  intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval)); |
| 3088 |  memset(intervals, 0, sizeof(zend_lifetime_interval*) * ssa->vars_count); |
| 3089 |  list = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval) * count); |
| 3090 |  j = 0; |
| 3091 |  for (i = 0; i < ssa->vars_count; i++) { |
| 3092 |    if (start[i] >= 0 && end[i] >= 0) { |
| 3093 |      ZEND_ASSERT(j < count); |
| 3094 |      if ((flags[i] & ZREG_LOAD) && |
| 3095 |       (flags[i] & ZREG_LAST_USE) && |
| 3096 |       end[i] == ssa->vars[i].use_chain) { |
| 3097 |        /* skip life range with single use */ |
| 3098 |        continue; |
| 3099 |      } |
| 3100 |      intervals[i] = &list[j]; |
| 3101 |      list[j].ssa_var = i; |
| 3102 |      list[j].reg = ZREG_NONE; |
| 3103 |      list[j].flags = flags[i]; |
| 3104 |      list[j].range.start = start[i]; |
| 3105 |      list[j].range.end = end[i]; |
| 3106 |      list[j].range.next = NULL; |
| 3107 |      list[j].hint = NULL; |
| 3108 |      list[j].used_as_hint = NULL; |
| 3109 |      list[j].list_next = NULL; |
| 3110 |      j++; |
| 3111 |    } |
| 3112 |  } |
| 3113 |  count = j; |
| 3114 |  free_alloca(start, use_heap); |
| 3115 |  start = end = NULL; |
| 3116 | |
| 3117 |  if (!count) { |
| 3118 |    zend_arena_release(&CG(arena), checkpoint); |
| 3119 |    return NULL; |
| 3120 |  } |
| 3121 | |
| 3122 |  /* Add hints */ |
| 3123 |  if (parent_vars_count) { |
| 3124 |    i = trace_buffer->op_array->last_var; |
| 3125 |    if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) { |
| 3126 |      i += trace_buffer->op_array->T; |
| 3127 |    } |
| 3128 |    if ((uint32_t)i > parent_vars_count) { |
| 3129 |      i = parent_vars_count; |
| 3130 |    } |
| 3131 |    while (i > 0) { |
| 3132 |      i--; |
| 3133 |      if ( |
| 3134 |       |
| 3135 |       ) { |
| 3136 |        list[j].ssa_var = - 1; |
| 3137 |        list[j].reg = STACK_REG(parent_stack, i); |
| 3138 |        list[j].flags = 0; |
| 3139 |        list[j].range.start = -1; |
| 3140 |        list[j].range.end = -1; |
| 3141 |        list[j].range.next = NULL; |
| 3142 |        list[j].hint = NULL; |
| 3143 |        list[j].used_as_hint = NULL; |
| 3144 |        list[j].list_next = NULL; |
| 3145 |        intervals[i]->hint = &list[j]; |
| 3146 |        j++; |
| 3147 |      } |
| 3148 |    } |
| 3149 |  } |
| 3150 | |
| 3151 |  if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP |
| 3152 |   || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL |
| 3153 |   || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 3154 |    zend_ssa_phi *phi = ssa->blocks[1].phis; |
| 3155 | |
| 3156 |    while (phi) { |
| 3157 |      if (intervals[phi->ssa_var]) { |
| 3158 |        if (intervals[phi->sources[1]] |
| 3159 |         && (ssa->var_info[phi->sources[1]].type & MAY_BE_ANY) == |
| 3160 |            (ssa->var_info[phi->ssa_var].type & MAY_BE_ANY)) { |
| 3161 |          intervals[phi->sources[1]]->hint = intervals[phi->ssa_var]; |
| 3162 |        } |
| 3163 |      } |
| 3164 |      phi = phi->next; |
| 3165 |    } |
| 3166 |  } |
| 3167 | |
| 3168 |  for (i = 0; i < ssa->vars_count; i++) { |
| 3169 |    if (intervals[i] && !intervals[i]->hint) { |
| 3170 | |
| 3171 |      if (ssa->vars[i].definition >= 0) { |
| 3172 |        uint32_t line = ssa->vars[i].definition; |
| 3173 |        const zend_op *opline = ssa_opcodes[line]; |
| 3174 | |
| 3175 |        switch (opline->opcode) { |
| 3176 |          case ZEND_QM_ASSIGN: |
| 3177 |          case ZEND_POST_INC: |
| 3178 |          case ZEND_POST_DEC: |
| 3179 |            if (ssa->ops[line].op1_use >= 0 && |
| 3180 |             intervals[ssa->ops[line].op1_use] && |
| 3181 |             (i == ssa->ops[line].op1_def || |
| 3182 |             (i == ssa->ops[line].result_def && |
| 3183 |             (ssa->ops[line].op1_def < 0 || |
| 3184 |             !intervals[ssa->ops[line].op1_def])))) { |
| 3185 |              zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use); |
| 3186 |            } |
| 3187 |            break; |
| 3188 |          case ZEND_SEND_VAR: |
| 3189 |            if (opline->op2_type == IS_CONST) { |
| 3190 |              /* Named parameters not supported in JIT */ |
| 3191 |              break; |
| 3192 |            } |
| 3193 |          case ZEND_PRE_INC: |
| 3194 |          case ZEND_PRE_DEC: |
| 3195 |            if (i == ssa->ops[line].op1_def && |
| 3196 |             ssa->ops[line].op1_use >= 0 && |
| 3197 |             intervals[ssa->ops[line].op1_use]) { |
| 3198 |              zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use); |
| 3199 |            } |
| 3200 |            break; |
| 3201 |          case ZEND_ASSIGN: |
| 3202 |            if (ssa->ops[line].op2_use >= 0 && |
| 3203 |             intervals[ssa->ops[line].op2_use] && |
| 3204 |             (i == ssa->ops[line].op2_def || |
| 3205 |               (i == ssa->ops[line].op1_def && |
| 3206 |             (ssa->ops[line].op2_def < 0 || |
| 3207 |             !intervals[ssa->ops[line].op2_def])) || |
| 3208 |               (i == ssa->ops[line].result_def && |
| 3209 |             (ssa->ops[line].op2_def < 0 || |
| 3210 |             !intervals[ssa->ops[line].op2_def]) && |
| 3211 |             (ssa->ops[line].op1_def < 0 || |
| 3212 |             !intervals[ssa->ops[line].op1_def])))) { |
| 3213 |              zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use); |
| 3214 |            } |
| 3215 |            break; |
| 3216 |          case ZEND_SUB: |
| 3217 |          case ZEND_ADD: |
| 3218 |          case ZEND_MUL: |
| 3219 |          case ZEND_BW_OR: |
| 3220 |          case ZEND_BW_AND: |
| 3221 |          case ZEND_BW_XOR: |
| 3222 |            if (i == ssa->ops[line].result_def) { |
| 3223 |              if (ssa->ops[line].op1_use >= 0 && |
| 3224 |               intervals[ssa->ops[line].op1_use] && |
| 3225 |               ssa->ops[line].op1_use_chain < 0 && |
| 3226 |               !ssa->vars[ssa->ops[line].op1_use].phi_use_chain && |
| 3227 |               (ssa->var_info[i].type & MAY_BE_ANY) == |
| 3228 |               (ssa->var_info[ssa->ops[line].op1_use].type & MAY_BE_ANY)) { |
| 3229 | |
| 3230 |                zend_ssa_phi *phi = ssa->vars[ssa->ops[line].op1_use].definition_phi; |
| 3231 |                if (phi && |
| 3232 |                 intervals[phi->sources[1]] && |
| 3233 |                 intervals[phi->sources[1]]->hint == intervals[ssa->ops[line].op1_use]) { |
| 3234 |                  break; |
| 3235 |                } |
| 3236 |                zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use); |
| 3237 |              } else if (opline->opcode != ZEND_SUB && |
| 3238 |               ssa->ops[line].op2_use >= 0 && |
| 3239 |               intervals[ssa->ops[line].op2_use] && |
| 3240 |               ssa->ops[line].op2_use_chain < 0 && |
| 3241 |               !ssa->vars[ssa->ops[line].op2_use].phi_use_chain && |
| 3242 |               (ssa->var_info[i].type & MAY_BE_ANY) == |
| 3243 |               (ssa->var_info[ssa->ops[line].op2_use].type & MAY_BE_ANY)) { |
| 3244 | |
| 3245 |                zend_ssa_phi *phi = ssa->vars[ssa->ops[line].op2_use].definition_phi; |
| 3246 |                if (phi && |
| 3247 |                 intervals[phi->sources[1]] && |
| 3248 |                 intervals[phi->sources[1]]->hint == intervals[ssa->ops[line].op2_use]) { |
| 3249 |                  break; |
| 3250 |                } |
| 3251 |                zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use); |
| 3252 |              } |
| 3253 |            } |
| 3254 |            break; |
| 3255 |        } |
| 3256 |      } |
| 3257 |    } |
| 3258 |  } |
| 3259 | |
| 3260 |  list = zend_jit_sort_intervals(intervals, ssa->vars_count); |
| 3261 | |
| 3262 |  if (list) { |
| 3263 |    ival = list; |
| 3264 |    while (ival) { |
| 3265 |      if (ival->hint) { |
| 3266 |        ival->hint->used_as_hint = ival; |
| 3267 |      } |
| 3268 |      ival = ival->list_next; |
| 3269 |    } |
| 3270 |  } |
| 3271 | |
| 3272 |  if (list) { |
| 3273 |    if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) { |
| 3274 |      fprintf(stderr, "---- TRACE %d Live Ranges\n", ZEND_JIT_TRACE_NUM); |
| 3275 |      ival = list; |
| 3276 |      while (ival) { |
| 3277 |        zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival); |
| 3278 |        ival = ival->list_next; |
| 3279 |      } |
| 3280 |    } |
| 3281 |  } |
| 3282 | |
| 3283 |  /* Linear Scan Register Allocation (op_array is not actually used, only fn_flags matters) */ |
| 3284 |  list = zend_jit_linear_scan(&dummy_op_array, ssa_opcodes, ssa, list); |
| 3285 | |
| 3286 |  if (list) { |
| 3287 |    zend_lifetime_interval *ival, *next; |
| 3288 | |
| 3289 |    memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*)); |
| 3290 |    ival = list; |
| 3291 |    count = 0; |
| 3292 |    while (ival != NULL) { |
| 3293 |      ZEND_ASSERT(ival->reg != ZREG_NONE); |
| 3294 |      count++; |
| 3295 |      next = ival->list_next; |
| 3296 |      ival->list_next = intervals[ival->ssa_var]; |
| 3297 |      intervals[ival->ssa_var] = ival; |
| 3298 |      ival = next; |
| 3299 |    } |
| 3300 | |
| 3301 |    if (!count) { |
| 3302 |      zend_arena_release(&CG(arena), checkpoint); |
| 3303 |      return NULL; |
| 3304 |    } |
| 3305 | |
| 3306 |    /* Add LOAD flag to registers that can't reuse register from parent trace */ |
| 3307 |    if (parent_vars_count) { |
| 3308 |      i = trace_buffer->op_array->last_var; |
| 3309 |      if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) { |
| 3310 |        i += trace_buffer->op_array->T; |
| 3311 |      } |
| 3312 |      if ((uint32_t)i > parent_vars_count) { |
| 3313 |        i = parent_vars_count; |
| 3314 |      } |
| 3315 |      while (i > 0) { |
| 3316 |        i--; |
| 3317 |        if (intervals[i] && intervals[i]->reg != STACK_REG(parent_stack, i)) { |
| 3318 |          intervals[i]->flags |= ZREG_LOAD; |
| 3319 |        } |
| 3320 |      } |
| 3321 |    } |
| 3322 | |
| 3323 |    /* SSA resolution */ |
| 3324 |    if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP |
| 3325 |     || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL |
| 3326 |     || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 3327 |      zend_ssa_phi *phi = ssa->blocks[1].phis; |
| 3328 | |
| 3329 |      while (phi) { |
| 3330 |        int def = phi->ssa_var; |
| 3331 |        int use = phi->sources[1]; |
| 3332 | |
| 3333 |        if (intervals[def]) { |
| 3334 |          if (!intervals[use]) { |
| 3335 |            intervals[def]->flags |= ZREG_LOAD; |
| 3336 |            if ((intervals[def]->flags & ZREG_LAST_USE) |
| 3337 |             && ssa->vars[def].use_chain >= 0 |
| 3338 |             && ssa->vars[def].use_chain == intervals[def]->range.end) { |
| 3339 |              /* remove interval used once */ |
| 3340 |              intervals[def] = NULL; |
| 3341 |              count--; |
| 3342 |            } |
| 3343 |          } else if (intervals[def]->reg != intervals[use]->reg) { |
| 3344 |            intervals[def]->flags |= ZREG_LOAD; |
| 3345 |            if (ssa->vars[use].use_chain >= 0) { |
| 3346 |              intervals[use]->flags |= ZREG_STORE; |
| 3347 |            } else { |
| 3348 |              intervals[use] = NULL; |
| 3349 |              count--; |
| 3350 |            } |
| 3351 |          } else { |
| 3352 |            use = phi->sources[0]; |
| 3353 |            ZEND_ASSERT(!intervals[use]); |
| 3354 |            intervals[use] = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval)); |
| 3355 |            intervals[use]->ssa_var = phi->sources[0]; |
| 3356 |            intervals[use]->reg = intervals[def]->reg; |
| 3357 |            intervals[use]->flags = ZREG_LOAD; |
| 3358 |            intervals[use]->range.start = 0; |
| 3359 |            intervals[use]->range.end = 0; |
| 3360 |            intervals[use]->range.next = NULL; |
| 3361 |            intervals[use]->hint = NULL; |
| 3362 |            intervals[use]->used_as_hint = NULL; |
| 3363 |            intervals[use]->list_next = NULL; |
| 3364 |          } |
| 3365 |        } else if (intervals[use] |
| 3366 |            && (!ssa->vars[def].no_val |
| 3367 |              || ssa->var_info[def].type != ssa->var_info[use].type)) { |
| 3368 |          if (ssa->vars[use].use_chain >= 0) { |
| 3369 |            intervals[use]->flags |= ZREG_STORE; |
| 3370 |          } else { |
| 3371 |            intervals[use] = NULL; |
| 3372 |            count--; |
| 3373 |          } |
| 3374 |        } |
| 3375 |        phi = phi->next; |
| 3376 |      } |
| 3377 |    } |
| 3378 | |
| 3379 |    if (!count) { |
| 3380 |      zend_arena_release(&CG(arena), checkpoint); |
| 3381 |      return NULL; |
| 3382 |    } |
| 3383 | |
| 3384 |    if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) { |
| 3385 |      fprintf(stderr, "---- TRACE %d Allocated Live Ranges\n", ZEND_JIT_TRACE_NUM); |
| 3386 |      for (i = 0; i < ssa->vars_count; i++) { |
| 3387 |        ival = intervals[i]; |
| 3388 |        while (ival) { |
| 3389 |          zend_jit_dump_lifetime_interval(vars_op_array[ival->ssa_var], ssa, ival); |
| 3390 |          ival = ival->list_next; |
| 3391 |        } |
| 3392 |      } |
| 3393 |    } |
| 3394 | |
| 3395 |    return intervals; |
| 3396 |  } |
| 3397 | |
| 3398 |  zend_arena_release(&CG(arena), checkpoint); //??? |
| 3399 |  return NULL; |
| 3400 | } |
| 3401 | |
| 3402 | static void zend_jit_trace_clenup_stack(zend_jit_trace_stack *stack, const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, zend_lifetime_interval **ra) |
| 3403 | { |
| 3404 |  uint32_t line = ssa_op - ssa->ops; |
| 3405 | |
| 3406 |  if (ssa_op->op1_use >= 0 |
| 3407 |   && ra[ssa_op->op1_use] |
| 3408 |   && ra[ssa_op->op1_use]->range.end == line) { |
| 3409 |    SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); |
| 3410 |  } |
| 3411 |  if (ssa_op->op2_use >= 0 |
| 3412 |   && ra[ssa_op->op2_use] |
| 3413 |   && ra[ssa_op->op2_use]->range.end == line) { |
| 3414 |    SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op2.var), ZREG_NONE); |
| 3415 |  } |
| 3416 |  if (ssa_op->result_use >= 0 |
| 3417 |   && ra[ssa_op->result_use] |
| 3418 |   && ra[ssa_op->result_use]->range.end == line) { |
| 3419 |    SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE); |
| 3420 |  } |
| 3421 | } |
| 3422 | |
| 3423 | static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offset) |
| 3424 | { |
| 3425 |  zend_op *next_opline = (zend_op*)(opline + 1); |
| 3426 | |
| 3427 |  if (JIT_G(hot_return) && !ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags) { |
| 3428 |    ZEND_ASSERT(zend_jit_ret_trace_counter_handler != NULL); |
| 3429 |    if (!ZEND_OP_TRACE_INFO(next_opline, offset)->counter) { |
| 3430 |      ZEND_OP_TRACE_INFO(next_opline, offset)->counter = |
| 3431 |        &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; |
| 3432 |      ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; |
| 3433 |    } |
| 3434 |    ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN; |
| 3435 |    next_opline->handler = (const void*)zend_jit_ret_trace_counter_handler; |
| 3436 |  } |
| 3437 | } |
| 3438 | |
| 3439 | static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_ssa_op *ssa_op) |
| 3440 | { |
| 3441 |  int var = ssa_op->result_def; |
| 3442 |  int i; |
| 3443 |  int use = ssa->vars[var].use_chain; |
| 3444 |  const zend_op *opline; |
| 3445 | |
| 3446 |  if (use < 0 |
| 3447 |   || ssa->vars[var].phi_use_chain |
| 3448 |   || ssa->ops[use].op1_use != var |
| 3449 |   || ssa->ops[use].op1_use_chain != -1) { |
| 3450 |    return 0; |
| 3451 |  } |
| 3452 | |
| 3453 |  opline = ssa_opcodes[use]; |
| 3454 |  if (opline->opcode == ZEND_INIT_METHOD_CALL) { |
| 3455 |    return (opline->op2_type == IS_CONST && |
| 3456 |      Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING); |
| 3457 |  } else if (opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG) { |
| 3458 |    if (!JIT_G(current_frame) |
| 3459 |     || !JIT_G(current_frame)->call |
| 3460 |     || !JIT_G(current_frame)->call->func |
| 3461 |     || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { |
| 3462 |      return 0; |
| 3463 |    } |
| 3464 |  } else if (opline->opcode != ZEND_FETCH_OBJ_R |
| 3465 |      && opline->opcode != ZEND_FETCH_OBJ_IS |
| 3466 |      && opline->opcode != ZEND_FETCH_OBJ_W |
| 3467 |      && opline->opcode != ZEND_ASSIGN_OBJ |
| 3468 |      && opline->opcode != ZEND_ASSIGN_OBJ_OP |
| 3469 |      && opline->opcode != ZEND_PRE_INC_OBJ |
| 3470 |      && opline->opcode != ZEND_PRE_DEC_OBJ |
| 3471 |      && opline->opcode != ZEND_POST_INC_OBJ |
| 3472 |      && opline->opcode != ZEND_POST_DEC_OBJ) { |
| 3473 |    return 0; |
| 3474 |  } |
| 3475 | |
| 3476 |  if (opline->op2_type != IS_CONST |
| 3477 |   || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING |
| 3478 |   || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') { |
| 3479 |    return 0; |
| 3480 |  } |
| 3481 | |
| 3482 |  if (opline->opcode == ZEND_ASSIGN_OBJ_OP) { |
| 3483 |    if (opline->op1_type == IS_CV |
| 3484 |     && (opline+1)->op1_type == IS_CV |
| 3485 |     && ) { |
| 3486 |      /* skip $a->prop += $a; */ |
| 3487 |      return 0; |
| 3488 |    } |
| 3489 |    if (!zend_jit_supported_binary_op( |
| 3490 |        opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) { |
| 3491 |      return 0; |
| 3492 |    } |
| 3493 |  } |
| 3494 | |
| 3495 |  for (i = ssa->vars[var].definition; i < use; i++) { |
| 3496 |    if (ssa_opcodes[i]->opcode == ZEND_DO_UCALL |
| 3497 |     || ssa_opcodes[i]->opcode == ZEND_DO_FCALL_BY_NAME |
| 3498 |     || ssa_opcodes[i]->opcode == ZEND_DO_FCALL |
| 3499 |     || ssa_opcodes[i]->opcode == ZEND_INCLUDE_OR_EVAL) { |
| 3500 |      return 0; |
| 3501 |    } |
| 3502 |  } |
| 3503 | |
| 3504 |  return 1; |
| 3505 | } |
| 3506 | |
| 3507 | static int zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size) |
| 3508 | { |
| 3509 |  uint32_t i; |
| 3510 | |
| 3511 |  for (i = 0; i < stack_size; i++) { |
| 3512 |    if (STACK_REG(stack, i) != ZREG_NONE |
| 3513 |     && !(STACK_FLAGS(stack, i) & (ZREG_LOAD|ZREG_STORE))) { |
| 3514 |      return 1; |
| 3515 |    } |
| 3516 |  } |
| 3517 |  return 0; |
| 3518 | } |
| 3519 | |
| 3520 | static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num) |
| 3521 | { |
| 3522 |  const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline; |
| 3523 |  uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags; |
| 3524 |  uint32_t stack_size; |
| 3525 |  zend_jit_trace_stack *stack; |
| 3526 | |
| 3527 |  if (opline || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2))) { |
| 3528 |    return 1; |
| 3529 |  } |
| 3530 | |
| 3531 |  stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size; |
| 3532 |  stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset; |
| 3533 |  return zend_jit_trace_stack_needs_deoptimization(stack, stack_size); |
| 3534 | } |
| 3535 | |
| 3536 | static int zend_jit_trace_deoptimization(dasm_State **Dst, |
| 3537 | uint32_t flags, |
| 3538 | const zend_op *opline, |
| 3539 | zend_jit_trace_stack *parent_stack, |
| 3540 | int parent_vars_count, |
| 3541 | zend_ssa *ssa, |
| 3542 | zend_jit_trace_stack *stack, |
| 3543 | zend_lifetime_interval **ra, |
| 3544 | bool polymorphic_side_trace) |
| 3545 | { |
| 3546 |  int i; |
| 3547 |  bool has_constants = 0; |
| 3548 |  bool has_unsaved_vars = 0; |
| 3549 | |
| 3550 |  // TODO: Merge this loop with the following register LOAD loop to implement parallel move ??? |
| 3551 |  for (i = 0; i < parent_vars_count; i++) { |
| 3552 |    int8_t reg = STACK_REG(parent_stack, i); |
| 3553 | |
| 3554 |    if (reg != ZREG_NONE) { |
| 3555 |      if (reg < ZREG_NUM) { |
| 3556 |        if (ssa && ssa->vars[i].no_val) { |
| 3557 |          /* pass */ |
| 3558 |        } else if (ra && ra[i] && ra[i]->reg == reg) { |
| 3559 |          /* register already loaded by parent trace */ |
| 3560 |          if (stack) { |
| 3561 |            SET_STACK_REG_EX(stack, i, reg, STACK_FLAGS(parent_stack, i)); |
| 3562 |          } |
| 3563 |          has_unsaved_vars = 1; |
| 3564 |        } else { |
| 3565 |          uint8_t type = STACK_TYPE(parent_stack, i); |
| 3566 | |
| 3567 |          if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) |
| 3568 |           && !zend_jit_store_var(Dst, 1 << type, i, reg, |
| 3569 |              STACK_MEM_TYPE(parent_stack, i) != type)) { |
| 3570 |            return 0; |
| 3571 |          } |
| 3572 |          if (stack) { |
| 3573 |            SET_STACK_TYPE(stack, i, type, 1); |
| 3574 |          } |
| 3575 |        } |
| 3576 |      } else { |
| 3577 |        /* delay custom deoptimization instructions to prevent register clobbering */ |
| 3578 |        has_constants = 1; |
| 3579 |      } |
| 3580 |    } |
| 3581 |  } |
| 3582 | |
| 3583 |  if (has_unsaved_vars |
| 3584 |   && (has_constants |
| 3585 |   || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)))) { |
| 3586 |    for (i = 0; i < parent_vars_count; i++) { |
| 3587 |      int8_t reg = STACK_REG(parent_stack, i); |
| 3588 | |
| 3589 |      if (reg != ZREG_NONE) { |
| 3590 |        if (reg < ZREG_NUM) { |
| 3591 |          if (ssa && ssa->vars[i].no_val) { |
| 3592 |            /* pass */ |
| 3593 |          } else if (ra && ra[i] && ra[i]->reg == reg) { |
| 3594 |            uint8_t type = STACK_TYPE(parent_stack, i); |
| 3595 | |
| 3596 |           if (stack) { |
| 3597 |              SET_STACK_TYPE(stack, i, type, 1); |
| 3598 |            } |
| 3599 |            if (!(STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) |
| 3600 |             && !zend_jit_store_var(Dst, 1 << type, i, reg, |
| 3601 |                STACK_MEM_TYPE(parent_stack, i) != type)) { |
| 3602 |              return 0; |
| 3603 |            } |
| 3604 |          } |
| 3605 |        } |
| 3606 |      } |
| 3607 |    } |
| 3608 |  } |
| 3609 | |
| 3610 |  if (has_constants) { |
| 3611 |    for (i = 0; i < parent_vars_count; i++) { |
| 3612 |      int8_t reg = STACK_REG(parent_stack, i); |
| 3613 | |
| 3614 |      if (reg != ZREG_NONE) { |
| 3615 |        if (reg < ZREG_NUM) { |
| 3616 |          /* pass */ |
| 3617 |        } else if (reg == ZREG_THIS) { |
| 3618 |          if (polymorphic_side_trace) { |
| 3619 |            ssa->var_info[i].delayed_fetch_this = 1; |
| 3620 |            if (stack) { |
| 3621 |              SET_STACK_REG(stack, i, ZREG_THIS); |
| 3622 |            } |
| 3623 |          } else if (!zend_jit_load_this(Dst, EX_NUM_TO_VAR(i))) { |
| 3624 |            return 0; |
| 3625 |          } |
| 3626 |        } else { |
| 3627 |          if (reg == ZREG_ZVAL_COPY_GPR0 |
| 3628 |           &&!zend_jit_escape_if_undef_r0(Dst, i, flags, opline)) { |
| 3629 |            return 0; |
| 3630 |          } |
| 3631 |          if (!zend_jit_store_const(Dst, i, reg)) { |
| 3632 |            return 0; |
| 3633 |          } |
| 3634 |        } |
| 3635 |      } |
| 3636 |    } |
| 3637 |  } |
| 3638 | |
| 3639 |  if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { |
| 3640 |    if (!zend_jit_save_call_chain(Dst, -1)) { |
| 3641 |      return 0; |
| 3642 |    } |
| 3643 |  } |
| 3644 | |
| 3645 |  if (flags & ZEND_JIT_EXIT_FREE_OP2) { |
| 3646 |    const zend_op *op = opline - 1; |
| 3647 | |
| 3648 |    if (!zend_jit_free_op(Dst, op, -1, op->op2.var)) { |
| 3649 |      return 0; |
| 3650 |    } |
| 3651 |  } |
| 3652 | |
| 3653 |  if (flags & ZEND_JIT_EXIT_FREE_OP1) { |
| 3654 |    const zend_op *op = opline - 1; |
| 3655 | |
| 3656 |    if (!zend_jit_free_op(Dst, op, -1, op->op1.var)) { |
| 3657 |      return 0; |
| 3658 |    } |
| 3659 |  } |
| 3660 | |
| 3661 |  if (flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) { |
| 3662 |    if (!zend_jit_check_exception(Dst)) { |
| 3663 |      return 0; |
| 3664 |    } |
| 3665 |  } |
| 3666 | |
| 3667 |  if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) { |
| 3668 |    if (!zend_jit_free_trampoline(Dst)) { |
| 3669 |      return 0; |
| 3670 |    } |
| 3671 |  } |
| 3672 | |
| 3673 |  return 1; |
| 3674 | } |
| 3675 | |
| 3676 | static void zend_jit_trace_set_var_range(zend_ssa_var_info *info, zend_long min, zend_long max) |
| 3677 | { |
| 3678 |  info->has_range = 1; |
| 3679 |  info->range.min = min; |
| 3680 |  info->range.max = max; |
| 3681 |  info->range.underflow = 0; |
| 3682 |  info->range.overflow = 0; |
| 3683 | } |
| 3684 | |
| 3685 | static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, bool exit_if_true) |
| 3686 | { |
| 3687 |  zend_long op1_min, op1_max, op2_min, op2_max; |
| 3688 | |
| 3689 |  if ((OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG |
| 3690 |   || (OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG) { |
| 3691 |    return; |
| 3692 |  } |
| 3693 | |
| 3694 |  op1_min = OP1_MIN_RANGE(); |
| 3695 |  op1_max = OP1_MAX_RANGE(); |
| 3696 |  op2_min = OP2_MIN_RANGE(); |
| 3697 |  op2_max = OP2_MAX_RANGE(); |
| 3698 | |
| 3699 |  switch (opline->opcode) { |
| 3700 |    case ZEND_IS_EQUAL: |
| 3701 |    case ZEND_CASE: |
| 3702 |    case ZEND_IS_IDENTICAL: |
| 3703 |    case ZEND_CASE_STRICT: |
| 3704 |    case ZEND_IS_NOT_IDENTICAL: |
| 3705 |      if (!exit_if_true) { |
| 3706 |        /* op1 == op2 */ |
| 3707 |        if (ssa_op->op1_use >= 0) { |
| 3708 |          zend_jit_trace_set_var_range( |
| 3709 |            &ssa->var_info[ssa_op->op1_use], |
| 3710 |            MAX(op1_min, op2_min), |
| 3711 |            MIN(op1_max, op2_max)); |
| 3712 |        } |
| 3713 |        if (ssa_op->op2_use >= 0) { |
| 3714 |          zend_jit_trace_set_var_range( |
| 3715 |            &ssa->var_info[ssa_op->op2_use], |
| 3716 |            MAX(op2_min, op1_min), |
| 3717 |            MIN(op2_max, op1_max)); |
| 3718 |        } |
| 3719 |      } |
| 3720 |      break; |
| 3721 |    case ZEND_IS_NOT_EQUAL: |
| 3722 |      if (exit_if_true) { |
| 3723 |        /* op1 == op2 */ |
| 3724 |        if (ssa_op->op1_use >= 0) { |
| 3725 |          zend_jit_trace_set_var_range( |
| 3726 |            &ssa->var_info[ssa_op->op1_use], |
| 3727 |            MAX(op1_min, op2_min), |
| 3728 |            MIN(op1_max, op2_max)); |
| 3729 |        } |
| 3730 |        if (ssa_op->op2_use >= 0) { |
| 3731 |          zend_jit_trace_set_var_range( |
| 3732 |            &ssa->var_info[ssa_op->op2_use], |
| 3733 |            MAX(op2_min, op1_min), |
| 3734 |            MIN(op2_max, op1_max)); |
| 3735 |        } |
| 3736 |      } |
| 3737 |      break; |
| 3738 |    case ZEND_IS_SMALLER_OR_EQUAL: |
| 3739 |      if (!exit_if_true) { |
| 3740 |        /* op1 <= op2 */ |
| 3741 |        if (ssa_op->op1_use >= 0) { |
| 3742 |          zend_jit_trace_set_var_range( |
| 3743 |            &ssa->var_info[ssa_op->op1_use], |
| 3744 |            op1_min, |
| 3745 |            MIN(op1_max, op2_max)); |
| 3746 |        } |
| 3747 |        if (ssa_op->op2_use >= 0) { |
| 3748 |          zend_jit_trace_set_var_range( |
| 3749 |            &ssa->var_info[ssa_op->op2_use], |
| 3750 |            MAX(op2_min, op1_min), |
| 3751 |            op2_max); |
| 3752 |        } |
| 3753 |      } else { |
| 3754 |        /* op1 > op2 */ |
| 3755 |        if (ssa_op->op1_use >= 0) { |
| 3756 |          zend_jit_trace_set_var_range( |
| 3757 |            &ssa->var_info[ssa_op->op1_use], |
| 3758 |            op2_min != ZEND_LONG_MAX ? MAX(op1_min, op2_min + 1) : op1_min, |
| 3759 |            op1_max); |
| 3760 |        } |
| 3761 |        if (ssa_op->op2_use >= 0) { |
| 3762 |          zend_jit_trace_set_var_range( |
| 3763 |            &ssa->var_info[ssa_op->op2_use], |
| 3764 |            op2_min, |
| 3765 |            op2_max != ZEND_LONG_MIN ?MIN(op2_max, op1_max - 1) : op1_max); |
| 3766 |        } |
| 3767 |      } |
| 3768 |      break; |
| 3769 |    case ZEND_IS_SMALLER: |
| 3770 |      if (!exit_if_true) { |
| 3771 |        /* op1 < op2 */ |
| 3772 |        if (ssa_op->op1_use >= 0) { |
| 3773 |          zend_jit_trace_set_var_range( |
| 3774 |            &ssa->var_info[ssa_op->op1_use], |
| 3775 |            op1_min, |
| 3776 |            op2_max != ZEND_LONG_MIN ? MIN(op1_max, op2_max - 1) : op1_max); |
| 3777 |        } |
| 3778 |        if (ssa_op->op2_use >= 0) { |
| 3779 |          zend_jit_trace_set_var_range( |
| 3780 |            &ssa->var_info[ssa_op->op2_use], |
| 3781 |            op1_min != ZEND_LONG_MAX ? MAX(op2_min, op1_min + 1) : op2_min, |
| 3782 |            op2_max); |
| 3783 |        } |
| 3784 |      } else { |
| 3785 |        /* op1 >= op2 */ |
| 3786 |        if (ssa_op->op1_use >= 0) { |
| 3787 |          zend_jit_trace_set_var_range( |
| 3788 |            &ssa->var_info[ssa_op->op1_use], |
| 3789 |            MAX(op1_min, op2_min), |
| 3790 |            op1_max); |
| 3791 |        } |
| 3792 |        if (ssa_op->op2_use >= 0) { |
| 3793 |          zend_jit_trace_set_var_range( |
| 3794 |            &ssa->var_info[ssa_op->op2_use], |
| 3795 |            op2_min, |
| 3796 |            MIN(op2_max, op1_max)); |
| 3797 |        } |
| 3798 |      } |
| 3799 |      break; |
| 3800 |  } |
| 3801 | } |
| 3802 | |
| 3803 | static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array) |
| 3804 | { |
| 3805 |  zend_uchar prev_opcode; |
| 3806 | |
| 3807 |  if (opline->op1_type == IS_CONST |
| 3808 |   && Z_TYPE_P(RT_CONSTANT(opline, opline->op1)) == IS_LONG |
| 3809 |   && Z_LVAL_P(RT_CONSTANT(opline, opline->op1)) == 0) { |
| 3810 |    if (ssa_op->op2_use >= 0) { |
| 3811 |      if ((ssa_op-1)->op1_def == ssa_op->op2_use) { |
| 3812 |        ssa_op--; |
| 3813 |        opline = ssa_opcodes[ssa_op - ssa->ops]; |
| 3814 |        prev_opcode = opline->opcode; |
| 3815 |        if (prev_opcode == ZEND_PRE_INC |
| 3816 |         || prev_opcode == ZEND_PRE_DEC |
| 3817 |         || prev_opcode == ZEND_POST_INC |
| 3818 |         || prev_opcode == ZEND_POST_DEC) { |
| 3819 |          return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0; |
| 3820 |        } |
| 3821 |      } else if ((ssa_op-1)->result_def == ssa_op->op2_use) { |
| 3822 |        ssa_op--; |
| 3823 |        opline = ssa_opcodes[ssa_op - ssa->ops]; |
| 3824 |        prev_opcode = opline->opcode; |
| 3825 |        if (prev_opcode == ZEND_ADD |
| 3826 |         || prev_opcode == ZEND_SUB) { |
| 3827 |          return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 && |
| 3828 |            (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0; |
| 3829 |        } |
| 3830 |      } |
| 3831 |    } |
| 3832 |  } else if (opline->op2_type == IS_CONST |
| 3833 |   && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG |
| 3834 |   && Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) == 0) { |
| 3835 |    if (ssa_op->op1_use >= 0) { |
| 3836 |      if ((ssa_op-1)->op1_def == ssa_op->op1_use) { |
| 3837 |        ssa_op--; |
| 3838 |        opline = ssa_opcodes[ssa_op - ssa->ops]; |
| 3839 |        prev_opcode = opline->opcode; |
| 3840 |        if (prev_opcode == ZEND_PRE_INC |
| 3841 |         || prev_opcode == ZEND_PRE_DEC |
| 3842 |         || prev_opcode == ZEND_POST_INC |
| 3843 |         || prev_opcode == ZEND_POST_DEC) { |
| 3844 |          return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0; |
| 3845 |        } |
| 3846 |      } else if ((ssa_op-1)->result_def == ssa_op->op1_use) { |
| 3847 |        ssa_op--; |
| 3848 |        opline = ssa_opcodes[ssa_op - ssa->ops]; |
| 3849 |        prev_opcode = opline->opcode; |
| 3850 |        if (prev_opcode == ZEND_ADD |
| 3851 |         || prev_opcode == ZEND_SUB) { |
| 3852 |          return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 && |
| 3853 |            (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0; |
| 3854 |        } |
| 3855 |      } |
| 3856 |    } |
| 3857 |  } else { |
| 3858 |    const zend_ssa_op *prev_ssa_op = ssa_op - 1; |
| 3859 |    prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode; |
| 3860 | |
| 3861 |    if ((prev_opcode == ZEND_JMPZ || prev_opcode == ZEND_JMPNZ) |
| 3862 |     && prev_ssa_op != ssa->ops |
| 3863 |     && prev_ssa_op->op1_use >= 0 |
| 3864 |     && ) { |
| 3865 |      prev_ssa_op--; |
| 3866 |      prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode; |
| 3867 |    } |
| 3868 | |
| 3869 |    if (ssa_op->op1_use == prev_ssa_op->op1_use |
| 3870 |     && ssa_op->op2_use == prev_ssa_op->op2_use) { |
| 3871 |      if (prev_opcode == ZEND_IS_EQUAL |
| 3872 |       || prev_opcode == ZEND_IS_NOT_EQUAL |
| 3873 |       || prev_opcode == ZEND_IS_SMALLER |
| 3874 |       || prev_opcode == ZEND_IS_SMALLER_OR_EQUAL |
| 3875 |       || prev_opcode == ZEND_CASE |
| 3876 |       || prev_opcode == ZEND_IS_IDENTICAL |
| 3877 |       || prev_opcode == ZEND_IS_NOT_IDENTICAL |
| 3878 |       || prev_opcode == ZEND_CASE_STRICT) { |
| 3879 |        if (ssa_op->op1_use < 0) { |
| 3880 |          if (RT_CONSTANT(opline, opline->op1) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op1)) { |
| 3881 |            return 0; |
| 3882 |          } |
| 3883 |        } |
| 3884 |        if (ssa_op->op2_use < 0) { |
| 3885 |          if (RT_CONSTANT(opline, opline->op2) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op2)) { |
| 3886 |            return 0; |
| 3887 |          } |
| 3888 |        } |
| 3889 |        return 1; |
| 3890 |      } |
| 3891 |    } |
| 3892 |  } |
| 3893 |  return 0; |
| 3894 | } |
| 3895 | |
| 3896 | static bool zend_jit_trace_next_is_send_result(const zend_op *opline, |
| 3897 | zend_jit_trace_rec *p, |
| 3898 | zend_jit_trace_stack_frame *frame) |
| 3899 | { |
| 3900 |  if (opline->result_type == IS_TMP_VAR |
| 3901 |   && (p+1)->op == ZEND_JIT_TRACE_VM |
| 3902 |   && |
| 3903 |   && ((opline+1)->opcode == ZEND_SEND_VAL |
| 3904 |   || ((opline+1)->opcode == ZEND_SEND_VAL_EX |
| 3905 |   && frame |
| 3906 |   && frame->call |
| 3907 |   && frame->call->func |
| 3908 |   && !ARG_MUST_BE_SENT_BY_REF(frame->call->func, (opline+1)->op2.num))) |
| 3909 |   && (opline+1)->op1_type == IS_TMP_VAR |
| 3910 |   && (opline+1)->op2_type != IS_CONST /* Named parameters not supported in JIT */ |
| 3911 |   && ) { |
| 3912 | |
| 3913 |    if (frame->call && frame->call->func) { |
| 3914 |      uint8_t res_type = (p+1)->op1_type; |
| 3915 | |
| 3916 |      if (res_type != IS_UNKNOWN && !(res_type & IS_TRACE_REFERENCE) ) { |
| 3917 |        zend_jit_trace_send_type(opline+1, frame->call, res_type); |
| 3918 |      } |
| 3919 |    } |
| 3920 |    return 1; |
| 3921 |  } |
| 3922 |  return 0; |
| 3923 | } |
| 3924 | |
| 3925 | static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num) |
| 3926 | { |
| 3927 |  const void *handler = NULL; |
| 3928 |  dasm_State* dasm_state = NULL; |
| 3929 |  zend_script *script = NULL; |
| 3930 |  zend_lifetime_interval **ra = NULL; |
| 3931 |  zend_string *name = NULL; |
| 3932 |  void *checkpoint; |
| 3933 |  const zend_op_array *op_array; |
| 3934 |  zend_ssa *ssa, *op_array_ssa; |
| 3935 |  const zend_op **ssa_opcodes; |
| 3936 |  zend_jit_trace_rec *p; |
| 3937 |  zend_jit_op_array_trace_extension *jit_extension; |
| 3938 |  int num_op_arrays = 0; |
| 3939 |  zend_jit_trace_info *t; |
| 3940 |  const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS]; |
| 3941 |  zend_uchar smart_branch_opcode; |
| 3942 |  const void *exit_addr; |
| 3943 |  uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info; |
| 3944 |  bool send_result = 0; |
| 3945 |  bool skip_comparison; |
| 3946 |  zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr; |
| 3947 |  zend_class_entry *ce; |
| 3948 |  bool ce_is_instanceof; |
| 3949 |  bool on_this = 0; |
| 3950 |  bool delayed_fetch_this = 0; |
| 3951 |  bool avoid_refcounting = 0; |
| 3952 |  bool polymorphic_side_trace = |
| 3953 |    parent_trace && |
| 3954 |    (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL); |
| 3955 |  uint32_t i; |
| 3956 |  zend_jit_trace_stack_frame *frame, *top, *call; |
| 3957 |  zend_jit_trace_stack *stack; |
| 3958 |  zend_uchar res_type = IS_UNKNOWN; |
| 3959 |  const zend_op *opline, *orig_opline; |
| 3960 |  const zend_ssa_op *ssa_op, *orig_ssa_op; |
| 3961 |  int checked_stack; |
| 3962 |  int peek_checked_stack; |
| 3963 |  uint32_t frame_flags = 0; |
| 3964 | |
| 3965 |  JIT_G(current_trace) = trace_buffer; |
| 3966 | |
| 3967 |  checkpoint = zend_arena_checkpoint(CG(arena)); |
| 3968 | |
| 3969 |  ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays); |
| 3970 | |
| 3971 |  if (!ssa) { |
| 3972 |    goto jit_cleanup; |
| 3973 |  } |
| 3974 | |
| 3975 |  ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes; |
| 3976 | |
| 3977 |  /* Register allocation */ |
| 3978 |  if ( |
| 3979 |   ) { |
| 3980 |    ra = zend_jit_trace_allocate_registers(trace_buffer, ssa, parent_trace, exit_num); |
| 3981 |  } |
| 3982 | |
| 3983 |  p = trace_buffer; |
| 3984 |  ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START); |
| 3985 |  op_array = p->op_array; |
| 3986 |  frame = JIT_G(current_frame); |
| 3987 |  top = zend_jit_trace_call_frame(frame, op_array); |
| 3988 |  TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1); |
| 3989 |  frame->used_stack = checked_stack = peek_checked_stack = 0; |
| 3990 |  stack = frame->stack; |
| 3991 |  for (i = 0; i < op_array->last_var + op_array->T; i++) { |
| 3992 |    SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1); |
| 3993 |  } |
| 3994 | |
| 3995 |  opline = p[1].opline; |
| 3996 |  name = zend_jit_trace_name(op_array, opline->lineno); |
| 3997 |  p += ZEND_JIT_TRACE_START_REC_SIZE; |
| 3998 | |
| 3999 |  dasm_init(&dasm_state, DASM_MAXSECTION); |
| 4000 |  dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); |
| 4001 |  dasm_setup(&dasm_state, dasm_actions); |
| 4002 | |
| 4003 |  jit_extension = |
| 4004 |    (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 4005 |  op_array_ssa = &jit_extension->func_info.ssa; |
| 4006 | |
| 4007 |  dasm_growpc(&dasm_state, 2); /* =>0: loop header */ |
| 4008 |   /* =>1: end of code */ |
| 4009 | |
| 4010 |  zend_jit_align_func(&dasm_state); |
| 4011 |  if (!parent_trace) { |
| 4012 |    zend_jit_prologue(&dasm_state); |
| 4013 |  } |
| 4014 |  zend_jit_trace_begin(&dasm_state, ZEND_JIT_TRACE_NUM, |
| 4015 |    parent_trace ? &zend_jit_traces[parent_trace] : NULL, exit_num); |
| 4016 | |
| 4017 |  if (!parent_trace) { |
| 4018 |    zend_jit_set_last_valid_opline(opline); |
| 4019 |    zend_jit_track_last_valid_opline(); |
| 4020 |  } else { |
| 4021 |    if (zend_jit_traces[parent_trace].exit_info[exit_num].opline == NULL) { |
| 4022 |      zend_jit_trace_opline_guard(&dasm_state, opline); |
| 4023 |    } else { |
| 4024 |      zend_jit_reset_last_valid_opline(); |
| 4025 |    } |
| 4026 |  } |
| 4027 | |
| 4028 |  if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { |
| 4029 |    int last_var; |
| 4030 |    int parent_vars_count = 0; |
| 4031 |    zend_jit_trace_stack *parent_stack = NULL; |
| 4032 |    int used_stack = ((zend_tssa*)ssa)->used_stack; |
| 4033 | |
| 4034 |    if (used_stack > 0) { |
| 4035 |      peek_checked_stack = used_stack; |
| 4036 |      if (!zend_jit_stack_check(&dasm_state, opline, used_stack)) { |
| 4037 |        goto jit_failure; |
| 4038 |      } |
| 4039 |    } |
| 4040 | |
| 4041 |    if (parent_trace) { |
| 4042 |      parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size, |
| 4043 |        op_array->last_var + op_array->T); |
| 4044 |      if (parent_vars_count) { |
| 4045 |        parent_stack = |
| 4046 |          zend_jit_traces[parent_trace].stack_map + |
| 4047 |          zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset; |
| 4048 |      } |
| 4049 |    } |
| 4050 | |
| 4051 |    last_var = op_array->last_var; |
| 4052 |    if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) { |
| 4053 |      last_var += op_array->T; |
| 4054 |    } |
| 4055 | |
| 4056 |    for (i = 0; i < last_var; i++) { |
| 4057 |      uint32_t info = ssa->var_info[i].type; |
| 4058 | |
| 4059 |      if (!(info & MAY_BE_GUARD) && has_concrete_type(info)) { |
| 4060 |        uint8_t type, mem_type; |
| 4061 | |
| 4062 |        type = concrete_type(info); |
| 4063 |        if (i < parent_vars_count |
| 4064 |         && STACK_TYPE(parent_stack, i) == type) { |
| 4065 |          mem_type = STACK_MEM_TYPE(parent_stack, i); |
| 4066 |          if (mem_type != IS_UNKNOWN) { |
| 4067 |            SET_STACK_TYPE(stack, i, mem_type, 1); |
| 4068 |          } |
| 4069 |          SET_STACK_TYPE(stack, i, type, 0); |
| 4070 |        } else { |
| 4071 |          SET_STACK_TYPE(stack, i, type, 1); |
| 4072 |        } |
| 4073 |      } else if (ssa->vars[i].alias != NO_ALIAS) { |
| 4074 |        SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1); |
| 4075 |      } else if (i < parent_vars_count |
| 4076 |       && STACK_TYPE(parent_stack, i) != IS_UNKNOWN) { |
| 4077 |        /* This must be already handled by trace type inference */ |
| 4078 |        ZEND_UNREACHABLE(); |
| 4079 |        // SET_STACK_TYPE(stack, i, STACK_TYPE(parent_stack, i)); |
| 4080 |      } else if ((info & MAY_BE_GUARD) != 0 |
| 4081 |       && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP |
| 4082 |       || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL |
| 4083 |       || (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET |
| 4084 |       && (opline-1)->result_type == IS_VAR |
| 4085 |       && EX_VAR_TO_NUM((opline-1)->result.var) == i)) |
| 4086 |       && (ssa->vars[i].use_chain != -1 |
| 4087 |       || (ssa->vars[i].phi_use_chain |
| 4088 |       && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_GUARD)))) { |
| 4089 |        /* Check loop-invariant variable type */ |
| 4090 |        if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), concrete_type(info))) { |
| 4091 |          goto jit_failure; |
| 4092 |        } |
| 4093 |        info &= ~MAY_BE_GUARD; |
| 4094 |        ssa->var_info[i].type = info; |
| 4095 |        SET_STACK_TYPE(stack, i, concrete_type(info), 1); |
| 4096 |      } else if ( |
| 4097 |       |
| 4098 |       ) { |
| 4099 |        /* This must be already handled by trace type inference */ |
| 4100 |        ZEND_UNREACHABLE(); |
| 4101 |        // SET_STACK_TYPE(stack, i, IS_UNDEF, 1); |
| 4102 |      } |
| 4103 | |
| 4104 |      if ((info & MAY_BE_PACKED_GUARD) != 0 |
| 4105 |       && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP |
| 4106 |       || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL |
| 4107 |       || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) |
| 4108 |       && (ssa->vars[i].use_chain != -1 |
| 4109 |       || (ssa->vars[i].phi_use_chain |
| 4110 |       && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_PACKED_GUARD)))) { |
| 4111 |        if (!zend_jit_packed_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), info)) { |
| 4112 |          goto jit_failure; |
| 4113 |        } |
| 4114 |        info &= ~MAY_BE_PACKED_GUARD; |
| 4115 |        ssa->var_info[i].type = info; |
| 4116 |      } |
| 4117 |    } |
| 4118 | |
| 4119 |    if (parent_trace) { |
| 4120 |      /* Deoptimization */ |
| 4121 |      if (!zend_jit_trace_deoptimization(&dasm_state, |
| 4122 |          zend_jit_traces[parent_trace].exit_info[exit_num].flags, |
| 4123 |          zend_jit_traces[parent_trace].exit_info[exit_num].opline, |
| 4124 |          parent_stack, parent_vars_count, ssa, stack, ra, |
| 4125 |          polymorphic_side_trace)) { |
| 4126 |        goto jit_failure; |
| 4127 |      } |
| 4128 |    } |
| 4129 | |
| 4130 |    if (ra |
| 4131 |     && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_CALL |
| 4132 |     && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 4133 |      for (i = 0; i < last_var; i++) { |
| 4134 |        if (ra[i] |
| 4135 |         && (ra[i]->flags & ZREG_LOAD) != 0 |
| 4136 |         && ra[i]->reg != stack[i].reg) { |
| 4137 | |
| 4138 |          if ((ssa->var_info[i].type & MAY_BE_GUARD) != 0) { |
| 4139 |            uint8_t op_type; |
| 4140 | |
| 4141 |            ssa->var_info[i].type &= ~MAY_BE_GUARD; |
| 4142 |            op_type = concrete_type(ssa->var_info[i].type); |
| 4143 |            if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(i), op_type)) { |
| 4144 |              goto jit_failure; |
| 4145 |            } |
| 4146 |            SET_STACK_TYPE(stack, i, op_type, 1); |
| 4147 |          } |
| 4148 | |
| 4149 |          SET_STACK_REG_EX(stack, i, ra[i]->reg, ZREG_LOAD); |
| 4150 |          if (!zend_jit_load_var(&dasm_state, ssa->var_info[i].type, i, ra[i]->reg)) { |
| 4151 |            goto jit_failure; |
| 4152 |          } |
| 4153 |        } |
| 4154 |      } |
| 4155 |    } |
| 4156 |  } |
| 4157 | |
| 4158 |  if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP |
| 4159 |   || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL |
| 4160 |   || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 4161 | |
| 4162 |    zend_jit_label(&dasm_state, 0); /* start of of trace loop */ |
| 4163 | |
| 4164 |    if (ra) { |
| 4165 |      zend_ssa_phi *phi = ssa->blocks[1].phis; |
| 4166 | |
| 4167 |      while (phi) { |
| 4168 |        zend_lifetime_interval *ival = ra[phi->ssa_var]; |
| 4169 | |
| 4170 |        if (ival) { |
| 4171 |          if (ival->flags & ZREG_LOAD) { |
| 4172 |            uint32_t info = ssa->var_info[phi->ssa_var].type; |
| 4173 |            ZEND_ASSERT(ival->reg != ZREG_NONE); |
| 4174 | |
| 4175 |            if (info & MAY_BE_GUARD) { |
| 4176 |              if (!zend_jit_type_guard(&dasm_state, opline, EX_NUM_TO_VAR(phi->var), concrete_type(info))) { |
| 4177 |                goto jit_failure; |
| 4178 |              } |
| 4179 |              info &= ~MAY_BE_GUARD; |
| 4180 |              ssa->var_info[phi->ssa_var].type = info; |
| 4181 |              SET_STACK_TYPE(stack, phi->var, concrete_type(info), 1); |
| 4182 |            } |
| 4183 |            SET_STACK_REG_EX(stack, phi->var, ival->reg, ZREG_LOAD); |
| 4184 |            if (!zend_jit_load_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg)) { |
| 4185 |              goto jit_failure; |
| 4186 |            } |
| 4187 |          } else if (ival->flags & ZREG_STORE) { |
| 4188 |            ZEND_ASSERT(ival->reg != ZREG_NONE); |
| 4189 | |
| 4190 |            SET_STACK_REG_EX(stack, phi->var, ival->reg, ZREG_STORE); |
| 4191 |            if (!zend_jit_store_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg, |
| 4192 |                STACK_MEM_TYPE(stack, phi->var) != ssa->var_info[phi->ssa_var].type)) { |
| 4193 |              goto jit_failure; |
| 4194 |            } |
| 4195 |          } else { |
| 4196 |            /* Register has to be written back on side exit */ |
| 4197 |            SET_STACK_REG(stack, phi->var, ival->reg); |
| 4198 |          } |
| 4199 |        } |
| 4200 |        phi = phi->next; |
| 4201 |      } |
| 4202 |    } |
| 4203 | |
| 4204 | //    if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 4205 | //      if (ra && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) { |
| 4206 | //        uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); |
| 4207 | // |
| 4208 | //        timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point); |
| 4209 | //        if (!timeout_exit_addr) { |
| 4210 | //          goto jit_failure; |
| 4211 | //        } |
| 4212 | //      } |
| 4213 | //    } |
| 4214 | |
| 4215 |    if (ra && trace_buffer->stop != ZEND_JIT_TRACE_STOP_LOOP) { |
| 4216 |      int last_var = op_array->last_var; |
| 4217 | |
| 4218 |      if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) { |
| 4219 |        last_var += op_array->T; |
| 4220 |      } |
| 4221 |      for (i = 0; i < last_var; i++) { |
| 4222 |        if (ra && ra[i] && (ra[i]->flags & ZREG_LOAD) != 0) { |
| 4223 |          SET_STACK_REG_EX(stack, i, ra[i]->reg, ZREG_LOAD); |
| 4224 |          if (!zend_jit_load_var(&dasm_state, ssa->var_info[i].type, i, ra[i]->reg)) { |
| 4225 |            goto jit_failure; |
| 4226 |          } |
| 4227 |        } |
| 4228 |      } |
| 4229 |    } |
| 4230 |  } |
| 4231 | |
| 4232 |  ssa_op = (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) ? ssa->ops : NULL; |
| 4233 |  for (;;p++) { |
| 4234 |    if (p->op == ZEND_JIT_TRACE_VM) { |
| 4235 |      uint8_t op1_type = p->op1_type; |
| 4236 |      uint8_t op2_type = p->op2_type; |
| 4237 |      uint8_t op3_type = p->op3_type; |
| 4238 |      uint8_t orig_op1_type = op1_type; |
| 4239 |      uint8_t orig_op2_type = op2_type; |
| 4240 |      uint8_t val_type = IS_UNKNOWN; |
| 4241 |      bool op1_indirect; |
| 4242 |      zend_class_entry *op1_ce = NULL; |
| 4243 |      zend_class_entry *op2_ce = NULL; |
| 4244 | |
| 4245 |      opline = p->opline; |
| 4246 |      if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) { |
| 4247 |        op1_type = IS_UNKNOWN; |
| 4248 |      } |
| 4249 |      if (op1_type != IS_UNKNOWN) { |
| 4250 |        op1_type &= ~IS_TRACE_PACKED; |
| 4251 |      } |
| 4252 |      if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) { |
| 4253 |        op2_type = IS_UNKNOWN; |
| 4254 |      } |
| 4255 |      if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) { |
| 4256 |        op3_type = IS_UNKNOWN; |
| 4257 |      } |
| 4258 | |
| 4259 |      if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) { |
| 4260 |        op1_ce = (zend_class_entry*)(p+1)->ce; |
| 4261 |        p++; |
| 4262 |      } |
| 4263 |      if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) { |
| 4264 |        op2_ce = (zend_class_entry*)(p+1)->ce; |
| 4265 |        p++; |
| 4266 |      } |
| 4267 |      if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) { |
| 4268 |        val_type = (p+1)->op1_type; |
| 4269 |        p++; |
| 4270 |      } |
| 4271 | |
| 4272 |      frame_flags = 0; |
| 4273 | |
| 4274 |      switch (opline->opcode) { |
| 4275 |        case ZEND_INIT_FCALL: |
| 4276 |        case ZEND_INIT_FCALL_BY_NAME: |
| 4277 |        case ZEND_INIT_NS_FCALL_BY_NAME: |
| 4278 |        case ZEND_INIT_METHOD_CALL: |
| 4279 |        case ZEND_INIT_DYNAMIC_CALL: |
| 4280 |        case ZEND_INIT_STATIC_METHOD_CALL: |
| 4281 |        case ZEND_INIT_USER_CALL: |
| 4282 |        case ZEND_NEW: |
| 4283 |          frame->call_level++; |
| 4284 |      } |
| 4285 | |
| 4286 |      if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { |
| 4287 |        switch (opline->opcode) { |
| 4288 |          case ZEND_PRE_INC: |
| 4289 |          case ZEND_PRE_DEC: |
| 4290 |          case ZEND_POST_INC: |
| 4291 |          case ZEND_POST_DEC: |
| 4292 |            if (opline->op1_type != IS_CV) { |
| 4293 |              break; |
| 4294 |            } |
| 4295 |            op1_info = OP1_INFO(); |
| 4296 |            CHECK_OP1_TRACE_TYPE(); |
| 4297 |            if (!(op1_info & MAY_BE_LONG)) { |
| 4298 |              break; |
| 4299 |            } |
| 4300 |            if (opline->result_type != IS_UNUSED) { |
| 4301 |              res_use_info = zend_jit_trace_type_to_info( |
| 4302 |                STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))) |
| 4303 |                  & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE); |
| 4304 |              res_info = RES_INFO(); |
| 4305 |              res_addr = RES_REG_ADDR(); |
| 4306 |            } else { |
| 4307 |              res_use_info = -1; |
| 4308 |              res_info = -1; |
| 4309 |              res_addr = 0; |
| 4310 |            } |
| 4311 |            op1_def_info = OP1_DEF_INFO(); |
| 4312 |            if (op1_def_info & MAY_BE_GUARD |
| 4313 |             && !has_concrete_type(op1_def_info)) { |
| 4314 |              op1_def_info &= ~MAY_BE_GUARD; |
| 4315 |            } |
| 4316 |            if (!zend_jit_inc_dec(&dasm_state, opline, |
| 4317 |                op1_info, OP1_REG_ADDR(), |
| 4318 |                op1_def_info, OP1_DEF_REG_ADDR(), |
| 4319 |                res_use_info, res_info, |
| 4320 |                res_addr, |
| 4321 |                (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa), |
| 4322 |                zend_may_throw(opline, ssa_op, op_array, ssa))) { |
| 4323 |              goto jit_failure; |
| 4324 |            } |
| 4325 |            if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD) |
| 4326 |             && !(op1_info & MAY_BE_STRING)) { |
| 4327 |              ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD; |
| 4328 |              if (opline->result_type != IS_UNUSED) { |
| 4329 |                ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; |
| 4330 |              } |
| 4331 |            } |
| 4332 |            if ( |
| 4333 |             |
| 4334 |             ) { |
| 4335 |              ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; |
| 4336 |            } |
| 4337 |            goto done; |
| 4338 |          case ZEND_BW_OR: |
| 4339 |          case ZEND_BW_AND: |
| 4340 |          case ZEND_BW_XOR: |
| 4341 |          case ZEND_SL: |
| 4342 |          case ZEND_SR: |
| 4343 |          case ZEND_MOD: |
| 4344 |            op1_info = OP1_INFO(); |
| 4345 |            CHECK_OP1_TRACE_TYPE(); |
| 4346 |            op2_info = OP2_INFO(); |
| 4347 |            CHECK_OP2_TRACE_TYPE(); |
| 4348 |            if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { |
| 4349 |              break; |
| 4350 |            } |
| 4351 |            if (!(op1_info & MAY_BE_LONG) |
| 4352 |             || !(op2_info & MAY_BE_LONG)) { |
| 4353 |              break; |
| 4354 |            } |
| 4355 |            res_addr = RES_REG_ADDR(); |
| 4356 |            if (Z_MODE(res_addr) != IS_REG |
| 4357 |             && zend_jit_trace_next_is_send_result(opline, p, frame)) { |
| 4358 |              send_result = 1; |
| 4359 |              res_use_info = -1; |
| 4360 |              res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); |
| 4361 |              if (!zend_jit_reuse_ip(&dasm_state)) { |
| 4362 |                goto jit_failure; |
| 4363 |              } |
| 4364 |            } else { |
| 4365 |              res_use_info = zend_jit_trace_type_to_info( |
| 4366 |                STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))) |
| 4367 |                  & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE); |
| 4368 |            } |
| 4369 |            res_info = RES_INFO(); |
| 4370 |            if (!zend_jit_long_math(&dasm_state, opline, |
| 4371 |                op1_info, OP1_RANGE(), OP1_REG_ADDR(), |
| 4372 |                op2_info, OP2_RANGE(), OP2_REG_ADDR(), |
| 4373 |                res_use_info, res_info, res_addr, |
| 4374 |                zend_may_throw(opline, ssa_op, op_array, ssa))) { |
| 4375 |              goto jit_failure; |
| 4376 |            } |
| 4377 |            goto done; |
| 4378 |          case ZEND_ADD: |
| 4379 |          case ZEND_SUB: |
| 4380 |          case ZEND_MUL: |
| 4381 | //          case ZEND_DIV: // TODO: check for division by zero ??? |
| 4382 |            op1_info = OP1_INFO(); |
| 4383 |            op1_addr = OP1_REG_ADDR(); |
| 4384 |            op2_info = OP2_INFO(); |
| 4385 |            op2_addr = OP2_REG_ADDR(); |
| 4386 |            if (orig_op1_type != IS_UNKNOWN |
| 4387 |             && (orig_op1_type & IS_TRACE_REFERENCE) |
| 4388 |             && opline->op1_type == IS_CV |
| 4389 |             && (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_FCARG1) |
| 4390 |             && (orig_op2_type == IS_UNKNOWN || !(orig_op2_type & IS_TRACE_REFERENCE))) { |
| 4391 |              if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, |
| 4392 |                  !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) { |
| 4393 |                goto jit_failure; |
| 4394 |              } |
| 4395 |              if (ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { |
| 4396 |                ssa->var_info[ssa_op->op1_use].guarded_reference = 1; |
| 4397 |              } |
| 4398 |            } else { |
| 4399 |              CHECK_OP1_TRACE_TYPE(); |
| 4400 |            } |
| 4401 |            if (orig_op2_type != IS_UNKNOWN |
| 4402 |             && (orig_op2_type & IS_TRACE_REFERENCE) |
| 4403 |             && opline->op2_type == IS_CV |
| 4404 |             && (Z_MODE(op1_addr) != IS_REG || Z_REG(op1_addr) != ZREG_FCARG1) |
| 4405 |             && (orig_op1_type == IS_UNKNOWN || !(orig_op1_type & IS_TRACE_REFERENCE))) { |
| 4406 |              if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op2_type, &op2_info, &op2_addr, |
| 4407 |                  !ssa->var_info[ssa_op->op2_use].guarded_reference, 1)) { |
| 4408 |                goto jit_failure; |
| 4409 |              } |
| 4410 |              if (ssa->vars[ssa_op->op2_use].alias == NO_ALIAS) { |
| 4411 |                ssa->var_info[ssa_op->op2_use].guarded_reference = 1; |
| 4412 |              } |
| 4413 |            } else { |
| 4414 |              CHECK_OP2_TRACE_TYPE(); |
| 4415 |            } |
| 4416 |            if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { |
| 4417 |              break; |
| 4418 |            } |
| 4419 |            if ( |
| 4420 |             |
| 4421 |             ) { |
| 4422 |              /* pass */ |
| 4423 |            } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) || |
| 4424 |             !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { |
| 4425 |              break; |
| 4426 |            } |
| 4427 |            res_addr = RES_REG_ADDR(); |
| 4428 |            if (Z_MODE(res_addr) != IS_REG |
| 4429 |             && zend_jit_trace_next_is_send_result(opline, p, frame)) { |
| 4430 |              send_result = 1; |
| 4431 |              res_use_info = -1; |
| 4432 |              res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); |
| 4433 |              if (!zend_jit_reuse_ip(&dasm_state)) { |
| 4434 |                goto jit_failure; |
| 4435 |              } |
| 4436 |            } else { |
| 4437 |              res_use_info = zend_jit_trace_type_to_info( |
| 4438 |                STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))) |
| 4439 |                  & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE); |
| 4440 |            } |
| 4441 |            res_info = RES_INFO(); |
| 4442 |            if ( |
| 4443 |             |
| 4444 |             ) { |
| 4445 |              if (!zend_jit_add_arrays(&dasm_state, opline, op1_info, op1_addr, op2_info, op2_addr, res_addr)) { |
| 4446 |                goto jit_failure; |
| 4447 |              } |
| 4448 |            } else { |
| 4449 |              if (!zend_jit_math(&dasm_state, opline, |
| 4450 |                  op1_info, op1_addr, |
| 4451 |                  op2_info, op2_addr, |
| 4452 |                  res_use_info, res_info, res_addr, |
| 4453 |                   , |
| 4454 |                  zend_may_throw(opline, ssa_op, op_array, ssa))) { |
| 4455 |                goto jit_failure; |
| 4456 |              } |
| 4457 |              if (((res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD) |
| 4458 |               || (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD)) |
| 4459 |               && has_concrete_type(op1_info) |
| 4460 |               && has_concrete_type(op2_info)) { |
| 4461 |                ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; |
| 4462 |              } |
| 4463 |            } |
| 4464 |            goto done; |
| 4465 |          case ZEND_CONCAT: |
| 4466 |          case ZEND_FAST_CONCAT: |
| 4467 |            op1_info = OP1_INFO(); |
| 4468 |            CHECK_OP1_TRACE_TYPE(); |
| 4469 |            op2_info = OP2_INFO(); |
| 4470 |            CHECK_OP2_TRACE_TYPE(); |
| 4471 |            if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { |
| 4472 |              break; |
| 4473 |            } |
| 4474 |            if (!(op1_info & MAY_BE_STRING) || |
| 4475 |             !(op2_info & MAY_BE_STRING)) { |
| 4476 |              break; |
| 4477 |            } |
| 4478 |            res_addr = RES_REG_ADDR(); |
| 4479 |            if (zend_jit_trace_next_is_send_result(opline, p, frame)) { |
| 4480 |              send_result = 1; |
| 4481 |              res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); |
| 4482 |              if (!zend_jit_reuse_ip(&dasm_state)) { |
| 4483 |                goto jit_failure; |
| 4484 |              } |
| 4485 |            } |
| 4486 |            if (!zend_jit_concat(&dasm_state, opline, |
| 4487 |                op1_info, op2_info, res_addr, |
| 4488 |                zend_may_throw(opline, ssa_op, op_array, ssa))) { |
| 4489 |              goto jit_failure; |
| 4490 |            } |
| 4491 |            goto done; |
| 4492 |          case ZEND_ASSIGN_OP: |
| 4493 |            if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) { |
| 4494 |              break; |
| 4495 |            } |
| 4496 |            op1_info = OP1_INFO(); |
| 4497 |            CHECK_OP1_TRACE_TYPE(); |
| 4498 |            op2_info = OP2_INFO(); |
| 4499 |            CHECK_OP2_TRACE_TYPE(); |
| 4500 |            if (!zend_jit_supported_binary_op( |
| 4501 |                opline->extended_value, op1_info, op2_info)) { |
| 4502 |              break; |
| 4503 |            } |
| 4504 |            op1_def_info = OP1_DEF_INFO(); |
| 4505 |            if (op1_def_info & MAY_BE_GUARD |
| 4506 |             && !has_concrete_type(op1_def_info)) { |
| 4507 |              op1_def_info &= ~MAY_BE_GUARD; |
| 4508 |            } |
| 4509 |            if (!zend_jit_assign_op(&dasm_state, opline, |
| 4510 |                op1_info, op1_def_info, OP1_RANGE(), |
| 4511 |                op2_info, OP2_RANGE(), |
| 4512 |                 , |
| 4513 |                zend_may_throw(opline, ssa_op, op_array, ssa))) { |
| 4514 |              goto jit_failure; |
| 4515 |            } |
| 4516 |            if ( |
| 4517 |             |
| 4518 |             ) { |
| 4519 |              ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD; |
| 4520 |              if (opline->result_type != IS_UNUSED) { |
| 4521 |                ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; |
| 4522 |              } |
| 4523 |            } |
| 4524 |            goto done; |
| 4525 |          case ZEND_ASSIGN_DIM_OP: |
| 4526 |            if (opline->result_type != IS_UNUSED) { |
| 4527 |              break; |
| 4528 |            } |
| 4529 |            if (!zend_jit_supported_binary_op( |
| 4530 |                opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) { |
| 4531 |              break; |
| 4532 |            } |
| 4533 |            if (opline->op1_type == IS_CV |
| 4534 |             && (opline+1)->op1_type == IS_CV |
| 4535 |             && ) { |
| 4536 |              /* skip $a[x] += $a; */ |
| 4537 |              break; |
| 4538 |            } |
| 4539 |            op1_info = OP1_INFO(); |
| 4540 |            op1_addr = OP1_REG_ADDR(); |
| 4541 |            if (opline->op1_type == IS_VAR) { |
| 4542 |              if (orig_op1_type != IS_UNKNOWN |
| 4543 |               && (orig_op1_type & IS_TRACE_INDIRECT)) { |
| 4544 |                if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type, |
| 4545 |                    &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) { |
| 4546 |                  goto jit_failure; |
| 4547 |                } |
| 4548 |              } else { |
| 4549 |                break; |
| 4550 |              } |
| 4551 |            } |
| 4552 |            if (orig_op1_type != IS_UNKNOWN |
| 4553 |             && (orig_op1_type & IS_TRACE_REFERENCE)) { |
| 4554 |              if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, |
| 4555 |                  !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) { |
| 4556 |                goto jit_failure; |
| 4557 |              } |
| 4558 |              if (opline->op1_type == IS_CV |
| 4559 |               && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { |
| 4560 |                ssa->var_info[ssa_op->op1_def].guarded_reference = 1; |
| 4561 |              } |
| 4562 |            } else { |
| 4563 |              CHECK_OP1_TRACE_TYPE(); |
| 4564 |            } |
| 4565 |            op2_info = OP2_INFO(); |
| 4566 |            CHECK_OP2_TRACE_TYPE(); |
| 4567 |            op1_data_info = OP1_DATA_INFO(); |
| 4568 |            CHECK_OP1_DATA_TRACE_TYPE(); |
| 4569 |            op1_def_info = OP1_DEF_INFO(); |
| 4570 |            if (!zend_jit_assign_dim_op(&dasm_state, opline, |
| 4571 |                op1_info, op1_def_info, op1_addr, op2_info, |
| 4572 |                op1_data_info, OP1_DATA_RANGE(), val_type, |
| 4573 |                zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) { |
| 4574 |              goto jit_failure; |
| 4575 |            } |
| 4576 |            goto done; |
| 4577 |          case ZEND_PRE_INC_OBJ: |
| 4578 |          case ZEND_PRE_DEC_OBJ: |
| 4579 |          case ZEND_POST_INC_OBJ: |
| 4580 |          case ZEND_POST_DEC_OBJ: |
| 4581 |            if (opline->op2_type != IS_CONST |
| 4582 |             || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING |
| 4583 |             || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') { |
| 4584 |              break; |
| 4585 |            } |
| 4586 |            ce = NULL; |
| 4587 |            ce_is_instanceof = 0; |
| 4588 |            on_this = delayed_fetch_this = 0; |
| 4589 |            op1_indirect = 0; |
| 4590 |            if (opline->op1_type == IS_UNUSED) { |
| 4591 |              op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; |
| 4592 |              ce = op_array->scope; |
| 4593 |              ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; |
| 4594 |              op1_addr = 0; |
| 4595 |              on_this = 1; |
| 4596 |            } else { |
| 4597 |              if (ssa_op->op1_use >= 0) { |
| 4598 |                delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this; |
| 4599 |              } |
| 4600 |              op1_info = OP1_INFO(); |
| 4601 |              if (!(op1_info & MAY_BE_OBJECT)) { |
| 4602 |                break; |
| 4603 |              } |
| 4604 |              op1_addr = OP1_REG_ADDR(); |
| 4605 |              if (opline->op1_type == IS_VAR) { |
| 4606 |                if (orig_op1_type != IS_UNKNOWN |
| 4607 |                 && (orig_op1_type & IS_TRACE_INDIRECT)) { |
| 4608 |                  op1_indirect = 1; |
| 4609 |                  if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type, |
| 4610 |                      &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) { |
| 4611 |                    goto jit_failure; |
| 4612 |                  } |
| 4613 |                } |
| 4614 |              } |
| 4615 |              if (orig_op1_type != IS_UNKNOWN |
| 4616 |               && (orig_op1_type & IS_TRACE_REFERENCE)) { |
| 4617 |                if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, |
| 4618 |                    !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) { |
| 4619 |                  goto jit_failure; |
| 4620 |                } |
| 4621 |                if (opline->op1_type == IS_CV |
| 4622 |                 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { |
| 4623 |                  ssa->var_info[ssa_op->op1_def].guarded_reference = 1; |
| 4624 |                } |
| 4625 |              } else { |
| 4626 |                CHECK_OP1_TRACE_TYPE(); |
| 4627 |              } |
| 4628 |              if (!(op1_info & MAY_BE_OBJECT)) { |
| 4629 |                break; |
| 4630 |              } |
| 4631 |              if (ssa->var_info && ssa->ops) { |
| 4632 |                if (ssa_op->op1_use >= 0) { |
| 4633 |                  zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use; |
| 4634 |                  if (op1_ssa->ce && !op1_ssa->ce->create_object) { |
| 4635 |                    ce = op1_ssa->ce; |
| 4636 |                    ce_is_instanceof = op1_ssa->is_instanceof; |
| 4637 |                  } |
| 4638 |                } |
| 4639 |              } |
| 4640 |              if (delayed_fetch_this) { |
| 4641 |                on_this = 1; |
| 4642 |              } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) { |
| 4643 |                on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS; |
| 4644 |              } else if (op_array_ssa->ops |
| 4645 |               && op_array_ssa->vars |
| 4646 |                  && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0 |
| 4647 |                  && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) { |
| 4648 |                on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS; |
| 4649 |              } |
| 4650 |            } |
| 4651 |            if (!zend_jit_incdec_obj(&dasm_state, opline, op_array, ssa, ssa_op, |
| 4652 |                op1_info, op1_addr, |
| 4653 |                op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce, |
| 4654 |                val_type)) { |
| 4655 |              goto jit_failure; |
| 4656 |            } |
| 4657 |            goto done; |
| 4658 |          case ZEND_ASSIGN_OBJ_OP: |
| 4659 |            if (opline->result_type != IS_UNUSED) { |
| 4660 |              break; |
| 4661 |            } |
| 4662 |            if (opline->op2_type != IS_CONST |
| 4663 |             || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING |
| 4664 |             || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') { |
| 4665 |              break; |
| 4666 |            } |
| 4667 |            if (opline->op1_type == IS_CV |
| 4668 |             && (opline+1)->op1_type == IS_CV |
| 4669 |             && ) { |
| 4670 |              /* skip $a->prop += $a; */ |
| 4671 |              break; |
| 4672 |            } |
| 4673 |            if (!zend_jit_supported_binary_op( |
| 4674 |                opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) { |
| 4675 |              break; |
| 4676 |            } |
| 4677 |            ce = NULL; |
| 4678 |            ce_is_instanceof = 0; |
| 4679 |            on_this = delayed_fetch_this = 0; |
| 4680 |            op1_indirect = 0; |
| 4681 |            if (opline->op1_type == IS_UNUSED) { |
| 4682 |              op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; |
| 4683 |              ce = op_array->scope; |
| 4684 |              ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; |
| 4685 |              op1_addr = 0; |
| 4686 |              on_this = 1; |
| 4687 |            } else { |
| 4688 |              if (ssa_op->op1_use >= 0) { |
| 4689 |                delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this; |
| 4690 |              } |
| 4691 |              op1_info = OP1_INFO(); |
| 4692 |              if (!(op1_info & MAY_BE_OBJECT)) { |
| 4693 |                break; |
| 4694 |              } |
| 4695 |              op1_addr = OP1_REG_ADDR(); |
| 4696 |              if (opline->op1_type == IS_VAR) { |
| 4697 |                if (orig_op1_type != IS_UNKNOWN |
| 4698 |                 && (orig_op1_type & IS_TRACE_INDIRECT)) { |
| 4699 |                  op1_indirect = 1; |
| 4700 |                  if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type, |
| 4701 |                      &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) { |
| 4702 |                    goto jit_failure; |
| 4703 |                  } |
| 4704 |                } |
| 4705 |              } |
| 4706 |              if (orig_op1_type != IS_UNKNOWN |
| 4707 |               && (orig_op1_type & IS_TRACE_REFERENCE)) { |
| 4708 |                if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, |
| 4709 |                    !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) { |
| 4710 |                  goto jit_failure; |
| 4711 |                } |
| 4712 |                if (opline->op1_type == IS_CV |
| 4713 |                 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { |
| 4714 |                  ssa->var_info[ssa_op->op1_def].guarded_reference = 1; |
| 4715 |                } |
| 4716 |              } else { |
| 4717 |                CHECK_OP1_TRACE_TYPE(); |
| 4718 |              } |
| 4719 |              if (!(op1_info & MAY_BE_OBJECT)) { |
| 4720 |                break; |
| 4721 |              } |
| 4722 |              if (ssa->var_info && ssa->ops) { |
| 4723 |                if (ssa_op->op1_use >= 0) { |
| 4724 |                  zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use; |
| 4725 |                  if (op1_ssa->ce && !op1_ssa->ce->create_object) { |
| 4726 |                    ce = op1_ssa->ce; |
| 4727 |                    ce_is_instanceof = op1_ssa->is_instanceof; |
| 4728 |                  } |
| 4729 |                } |
| 4730 |              } |
| 4731 |              if (delayed_fetch_this) { |
| 4732 |                on_this = 1; |
| 4733 |              } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) { |
| 4734 |                on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS; |
| 4735 |              } else if (op_array_ssa->ops |
| 4736 |               && op_array_ssa->vars |
| 4737 |                  && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0 |
| 4738 |                  && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) { |
| 4739 |                on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS; |
| 4740 |              } |
| 4741 |            } |
| 4742 |            op1_data_info = OP1_DATA_INFO(); |
| 4743 |            CHECK_OP1_DATA_TRACE_TYPE(); |
| 4744 |            if (!zend_jit_assign_obj_op(&dasm_state, opline, op_array, ssa, ssa_op, |
| 4745 |                op1_info, op1_addr, op1_data_info, OP1_DATA_RANGE(), |
| 4746 |                op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce, |
| 4747 |                val_type)) { |
| 4748 |              goto jit_failure; |
| 4749 |            } |
| 4750 |            goto done; |
| 4751 |          case ZEND_ASSIGN_OBJ: |
| 4752 |            if (opline->op2_type != IS_CONST |
| 4753 |             || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING |
| 4754 |             || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') { |
| 4755 |              break; |
| 4756 |            } |
| 4757 |            ce = NULL; |
| 4758 |            ce_is_instanceof = 0; |
| 4759 |            on_this = delayed_fetch_this = 0; |
| 4760 |            op1_indirect = 0; |
| 4761 |            if (opline->op1_type == IS_UNUSED) { |
| 4762 |              op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; |
| 4763 |              ce = op_array->scope; |
| 4764 |              ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; |
| 4765 |              op1_addr = 0; |
| 4766 |              on_this = 1; |
| 4767 |            } else { |
| 4768 |              if (ssa_op->op1_use >= 0) { |
| 4769 |                delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this; |
| 4770 |              } |
| 4771 |              op1_info = OP1_INFO(); |
| 4772 |              if (!(op1_info & MAY_BE_OBJECT)) { |
| 4773 |                break; |
| 4774 |              } |
| 4775 |              op1_addr = OP1_REG_ADDR(); |
| 4776 |              if (opline->op1_type == IS_VAR) { |
| 4777 |                if (orig_op1_type != IS_UNKNOWN |
| 4778 |                 && (orig_op1_type & IS_TRACE_INDIRECT)) { |
| 4779 |                  op1_indirect = 1; |
| 4780 |                  if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type, |
| 4781 |                      &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) { |
| 4782 |                    goto jit_failure; |
| 4783 |                  } |
| 4784 |                } |
| 4785 |              } |
| 4786 |              if (orig_op1_type != IS_UNKNOWN |
| 4787 |               && (orig_op1_type & IS_TRACE_REFERENCE)) { |
| 4788 |                if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, |
| 4789 |                    !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) { |
| 4790 |                  goto jit_failure; |
| 4791 |                } |
| 4792 |                if (opline->op1_type == IS_CV |
| 4793 |                 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { |
| 4794 |                  ssa->var_info[ssa_op->op1_def].guarded_reference = 1; |
| 4795 |                } |
| 4796 |              } else { |
| 4797 |                CHECK_OP1_TRACE_TYPE(); |
| 4798 |              } |
| 4799 |              if (!(op1_info & MAY_BE_OBJECT)) { |
| 4800 |                break; |
| 4801 |              } |
| 4802 |              if (ssa->var_info && ssa->ops) { |
| 4803 |                if (ssa_op->op1_use >= 0) { |
| 4804 |                  zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use; |
| 4805 |                  if (op1_ssa->ce && !op1_ssa->ce->create_object) { |
| 4806 |                    ce = op1_ssa->ce; |
| 4807 |                    ce_is_instanceof = op1_ssa->is_instanceof; |
| 4808 |                  } |
| 4809 |                } |
| 4810 |              } |
| 4811 |              if (delayed_fetch_this) { |
| 4812 |                on_this = 1; |
| 4813 |              } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) { |
| 4814 |                on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS; |
| 4815 |              } else if (op_array_ssa->ops |
| 4816 |               && op_array_ssa->vars |
| 4817 |                  && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0 |
| 4818 |                  && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) { |
| 4819 |                on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS; |
| 4820 |              } |
| 4821 |            } |
| 4822 |            op1_data_info = OP1_DATA_INFO(); |
| 4823 |            CHECK_OP1_DATA_TRACE_TYPE(); |
| 4824 |            if (!zend_jit_assign_obj(&dasm_state, opline, op_array, ssa, ssa_op, |
| 4825 |                op1_info, op1_addr, op1_data_info, |
| 4826 |                op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce, |
| 4827 |                val_type, |
| 4828 |                zend_may_throw(opline, ssa_op, op_array, ssa))) { |
| 4829 |              goto jit_failure; |
| 4830 |            } |
| 4831 |            if ((opline+1)->op1_type == IS_CV |
| 4832 |             && (ssa_op+1)->op1_def >= 0 |
| 4833 |             && .alias == NO_ALIAS) { |
| 4834 |              ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference; |
| 4835 |            } |
| 4836 |            goto done; |
| 4837 |          case ZEND_ASSIGN_DIM: |
| 4838 |            op1_info = OP1_INFO(); |
| 4839 |            op1_addr = OP1_REG_ADDR(); |
| 4840 |            if (opline->op1_type == IS_CV |
| 4841 |             && (opline+1)->op1_type == IS_CV |
| 4842 |             && ) { |
| 4843 |              /* skip $a[x] = $a; */ |
| 4844 |              break; |
| 4845 |            } |
| 4846 |            if (opline->op1_type == IS_VAR) { |
| 4847 |              if ( |
| 4848 |               |
| 4849 |               ) { |
| 4850 |                if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type, |
| 4851 |                    &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) { |
| 4852 |                  goto jit_failure; |
| 4853 |                } |
| 4854 |              } else { |
| 4855 |                break; |
| 4856 |              } |
| 4857 |            } |
| 4858 |            if (orig_op1_type != IS_UNKNOWN |
| 4859 |             && (orig_op1_type & IS_TRACE_REFERENCE)) { |
| 4860 |              if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, |
| 4861 |                  !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) { |
| 4862 |                goto jit_failure; |
| 4863 |              } |
| 4864 |              if (opline->op1_type == IS_CV |
| 4865 |               && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { |
| 4866 |                ssa->var_info[ssa_op->op1_def].guarded_reference = 1; |
| 4867 |              } |
| 4868 |            } else { |
| 4869 |              CHECK_OP1_TRACE_TYPE(); |
| 4870 |            } |
| 4871 |            op2_info = OP2_INFO(); |
| 4872 |            CHECK_OP2_TRACE_TYPE(); |
| 4873 |            op1_data_info = OP1_DATA_INFO(); |
| 4874 |            CHECK_OP1_DATA_TRACE_TYPE(); |
| 4875 |            if (!zend_jit_assign_dim(&dasm_state, opline, |
| 4876 |                op1_info, op1_addr, op2_info, op1_data_info, val_type, |
| 4877 |                zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) { |
| 4878 |              goto jit_failure; |
| 4879 |            } |
| 4880 |            if ((opline+1)->op1_type == IS_CV |
| 4881 |             && (ssa_op+1)->op1_def >= 0 |
| 4882 |             && .alias == NO_ALIAS) { |
| 4883 |              ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference; |
| 4884 |            } |
| 4885 |            goto done; |
| 4886 |          case ZEND_ASSIGN: |
| 4887 |            if (opline->op1_type != IS_CV) { |
| 4888 |              break; |
| 4889 |            } |
| 4890 |            op2_addr = OP2_REG_ADDR(); |
| 4891 |            op2_info = OP2_INFO(); |
| 4892 |            if (ra |
| 4893 |             && ssa_op->op2_def >= 0 |
| 4894 |             && (!ssa->vars[ssa_op->op2_def].no_val |
| 4895 |             || (zend_jit_trace_type_to_info(STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var))) & MAY_BE_ANY) != |
| 4896 |             (op2_info & MAY_BE_ANY))) { |
| 4897 |              op2_def_addr = OP2_DEF_REG_ADDR(); |
| 4898 |            } else { |
| 4899 |              op2_def_addr = op2_addr; |
| 4900 |            } |
| 4901 |            CHECK_OP2_TRACE_TYPE(); |
| 4902 |            op1_info = OP1_INFO(); |
| 4903 |            op1_def_info = OP1_DEF_INFO(); |
| 4904 |            if (op1_type != IS_UNKNOWN && (op1_info & MAY_BE_GUARD)) { |
| 4905 |              if ( |
| 4906 |               ) { |
| 4907 |                if (!zend_jit_scalar_type_guard(&dasm_state, opline, opline->op1.var)) { |
| 4908 |                  goto jit_failure; |
| 4909 |                } |
| 4910 |                op1_info &= ~(MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD); |
| 4911 |              } else { |
| 4912 |                CHECK_OP1_TRACE_TYPE(); |
| 4913 |              } |
| 4914 |            } |
| 4915 |            op1_addr = OP1_REG_ADDR(); |
| 4916 |            op1_def_addr = OP1_DEF_REG_ADDR(); |
| 4917 |            if (Z_MODE(op1_def_addr) != IS_REG && |
| 4918 |                STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) != |
| 4919 |                STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var))) { |
| 4920 |              /* type may be not set */ |
| 4921 |              op1_info |= MAY_BE_NULL; |
| 4922 |            } |
| 4923 |            if (orig_op1_type != IS_UNKNOWN) { |
| 4924 |              if (orig_op1_type & IS_TRACE_REFERENCE) { |
| 4925 |                if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, |
| 4926 |                    !ssa->var_info[ssa_op->op1_use].guarded_reference, 0)) { |
| 4927 |                  goto jit_failure; |
| 4928 |                } |
| 4929 |                if (opline->op1_type == IS_CV |
| 4930 |                 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { |
| 4931 |                  ssa->var_info[ssa_op->op1_def].guarded_reference = 1; |
| 4932 |                } |
| 4933 |                if (opline->result_type == IS_UNUSED) { |
| 4934 |                  res_addr = 0; |
| 4935 |                } else { |
| 4936 |                  res_addr = RES_REG_ADDR(); |
| 4937 |                } |
| 4938 |                if (!zend_jit_assign_to_typed_ref(&dasm_state, opline, opline->op2_type, op2_addr, res_addr, 1)) { |
| 4939 |                  goto jit_failure; |
| 4940 |                } |
| 4941 |                op1_def_addr = op1_addr; |
| 4942 |                op1_def_info &= ~MAY_BE_REF; |
| 4943 |              } else if (op1_info & MAY_BE_REF) { |
| 4944 |                if (!zend_jit_noref_guard(&dasm_state, opline, op1_addr)) { |
| 4945 |                  goto jit_failure; |
| 4946 |                } |
| 4947 |                op1_info &= ~MAY_BE_REF; |
| 4948 |                op1_def_info &= ~MAY_BE_REF; |
| 4949 |              } |
| 4950 |            } |
| 4951 |            if (opline->result_type == IS_UNUSED) { |
| 4952 |              res_addr = 0; |
| 4953 |              res_info = -1; |
| 4954 |            } else { |
| 4955 |              res_addr = RES_REG_ADDR(); |
| 4956 |              res_info = RES_INFO(); |
| 4957 |              if (Z_MODE(res_addr) != IS_REG |
| 4958 |               && zend_jit_trace_next_is_send_result(opline, p, frame)) { |
| 4959 |                send_result = 1; |
| 4960 |                res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var); |
| 4961 |                if (!zend_jit_reuse_ip(&dasm_state)) { |
| 4962 |                  goto jit_failure; |
| 4963 |                } |
| 4964 |              } |
| 4965 |            } |
| 4966 |            if (!zend_jit_assign(&dasm_state, opline, |
| 4967 |                op1_info, op1_addr, |
| 4968 |                op1_def_info, op1_def_addr, |
| 4969 |                op2_info, op2_addr, op2_def_addr, |
| 4970 |                res_info, res_addr, |
| 4971 |                zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) { |
| 4972 |              goto jit_failure; |
| 4973 |            } |
| 4974 |            if (opline->op2_type == IS_CV |
| 4975 |             && ssa_op->op2_def >= 0 |
| 4976 |             && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS) { |
| 4977 |              ssa->var_info[ssa_op->op2_def].guarded_reference = ssa->var_info[ssa_op->op2_use].guarded_reference; |
| 4978 |            } |
| 4979 |            goto done; |
| 4980 |          case ZEND_CAST: |
| 4981 |            if (opline->extended_value != op1_type) { |
| 4982 |              break; |
| 4983 |            } |
| 4984 |            ZEND_FALLTHROUGH; |
| 4985 |          case ZEND_QM_ASSIGN: |
| 4986 |            op1_addr = OP1_REG_ADDR(); |
| 4987 |            if (ra |
| 4988 |             && ssa_op->op1_def >= 0 |
| 4989 |             && !ssa->vars[ssa_op->op1_def].no_val) { |
| 4990 |              op1_def_addr = OP1_DEF_REG_ADDR(); |
| 4991 |            } else { |
| 4992 |              op1_def_addr = op1_addr; |
| 4993 |            } |
| 4994 |            op1_info = OP1_INFO(); |
| 4995 |            CHECK_OP1_TRACE_TYPE(); |
| 4996 |            res_info = RES_INFO(); |
| 4997 |            res_use_info = zend_jit_trace_type_to_info( |
| 4998 |              STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))) |
| 4999 |                & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE); |
| 5000 |            res_addr = RES_REG_ADDR(); |
| 5001 |            if (Z_MODE(res_addr) != IS_REG && |
| 5002 |                STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)) != |
| 5003 |                STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))) { |
| 5004 |              /* type may be not set */ |
| 5005 |              res_use_info |= MAY_BE_NULL; |
| 5006 |            } |
| 5007 |            if (!zend_jit_qm_assign(&dasm_state, opline, |
| 5008 |                op1_info, op1_addr, op1_def_addr, |
| 5009 |                res_use_info, res_info, res_addr)) { |
| 5010 |              goto jit_failure; |
| 5011 |            } |
| 5012 |            if (opline->op1_type == IS_CV |
| 5013 |             && ssa_op->op1_def >= 0 |
| 5014 |             && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { |
| 5015 |              ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference; |
| 5016 |            } |
| 5017 |            goto done; |
| 5018 |          case ZEND_INIT_FCALL: |
| 5019 |          case ZEND_INIT_FCALL_BY_NAME: |
| 5020 |          case ZEND_INIT_NS_FCALL_BY_NAME: |
| 5021 |            frame_flags = TRACE_FRAME_MASK_NESTED; |
| 5022 |            if (!zend_jit_init_fcall(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) { |
| 5023 |              goto jit_failure; |
| 5024 |            } |
| 5025 |            goto done; |
| 5026 |          case ZEND_SEND_VAL: |
| 5027 |          case ZEND_SEND_VAL_EX: |
| 5028 |            if (opline->op2_type == IS_CONST) { |
| 5029 |              /* Named parameters not supported in JIT */ |
| 5030 |              break; |
| 5031 |            } |
| 5032 |            if (opline->opcode == ZEND_SEND_VAL_EX |
| 5033 |             && opline->op2.num > MAX_ARG_FLAG_NUM) { |
| 5034 |              break; |
| 5035 |            } |
| 5036 |            op1_info = OP1_INFO(); |
| 5037 |            CHECK_OP1_TRACE_TYPE(); |
| 5038 |            if (!zend_jit_send_val(&dasm_state, opline, |
| 5039 |                op1_info, OP1_REG_ADDR())) { |
| 5040 |              goto jit_failure; |
| 5041 |            } |
| 5042 |            if (frame->call && frame->call->func) { |
| 5043 |              if (opline->op1_type == IS_CONST) { |
| 5044 |                zend_jit_trace_send_type(opline, frame->call, Z_TYPE_P(RT_CONSTANT(opline, opline->op1))); |
| 5045 |              } else if (op1_type != IS_UNKNOWN) { |
| 5046 |                if (op1_type == IS_UNDEF) { |
| 5047 |                  op1_type = IS_NULL; |
| 5048 |                } |
| 5049 |                zend_jit_trace_send_type(opline, frame->call, op1_type); |
| 5050 |              } |
| 5051 |            } |
| 5052 |            goto done; |
| 5053 |          case ZEND_SEND_REF: |
| 5054 |            if (opline->op2_type == IS_CONST) { |
| 5055 |              /* Named parameters not supported in JIT */ |
| 5056 |              break; |
| 5057 |            } |
| 5058 |            op1_info = OP1_INFO(); |
| 5059 |            if (!zend_jit_send_ref(&dasm_state, opline, op_array, |
| 5060 |                op1_info, 0)) { |
| 5061 |              goto jit_failure; |
| 5062 |            } |
| 5063 |            if (opline->op1_type == IS_CV |
| 5064 |             && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { |
| 5065 |              ssa->var_info[ssa_op->op1_def].guarded_reference = 1; |
| 5066 |            } |
| 5067 |            goto done; |
| 5068 |          case ZEND_SEND_VAR: |
| 5069 |          case ZEND_SEND_VAR_EX: |
| 5070 |          case ZEND_SEND_VAR_NO_REF: |
| 5071 |          case ZEND_SEND_VAR_NO_REF_EX: |
| 5072 |          case ZEND_SEND_FUNC_ARG: |
| 5073 |            if (opline->op2_type == IS_CONST) { |
| 5074 |              /* Named parameters not supported in JIT */ |
| 5075 |              break; |
| 5076 |            } |
| 5077 |            if ((opline->opcode == ZEND_SEND_VAR_EX |
| 5078 |             || opline->opcode == ZEND_SEND_VAR_NO_REF_EX) |
| 5079 |             && opline->op2.num > MAX_ARG_FLAG_NUM) { |
| 5080 |              break; |
| 5081 |            } |
| 5082 |            op1_addr = OP1_REG_ADDR(); |
| 5083 |            if (ra |
| 5084 |             && ssa_op->op1_def >= 0 |
| 5085 |             && !ssa->vars[ssa_op->op1_def].no_val) { |
| 5086 |              op1_def_addr = OP1_DEF_REG_ADDR(); |
| 5087 |            } else { |
| 5088 |              op1_def_addr = op1_addr; |
| 5089 |            } |
| 5090 |            op1_info = OP1_INFO(); |
| 5091 |            CHECK_OP1_TRACE_TYPE(); |
| 5092 |            if (!zend_jit_send_var(&dasm_state, opline, op_array, |
| 5093 |                op1_info, op1_addr, op1_def_addr)) { |
| 5094 |              goto jit_failure; |
| 5095 |            } |
| 5096 |            if (opline->op1_type == IS_CV |
| 5097 |             && ssa_op->op1_def >= 0 |
| 5098 |             && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { |
| 5099 |              ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference; |
| 5100 |            } |
| 5101 |            if (frame->call && frame->call->func) { |
| 5102 |              if ((opline->opcode == ZEND_SEND_VAR_EX |
| 5103 |               || opline->opcode == ZEND_SEND_FUNC_ARG) |
| 5104 |               && ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) { |
| 5105 |                goto done; |
| 5106 |              } |
| 5107 |              if (op1_type != IS_UNKNOWN) { |
| 5108 |                if (op1_type == IS_UNDEF) { |
| 5109 |                  op1_type = IS_NULL; |
| 5110 |                } |
| 5111 |                zend_jit_trace_send_type(opline, frame->call, op1_type); |
| 5112 |              } |
| 5113 |            } |
| 5114 |            goto done; |
| 5115 |          case ZEND_CHECK_FUNC_ARG: |
| 5116 |            if (!JIT_G(current_frame) |
| 5117 |             || !JIT_G(current_frame)->call |
| 5118 |             || !JIT_G(current_frame)->call->func) { |
| 5119 |              break; |
| 5120 |            } |
| 5121 |            if ( |
| 5122 |             ) { |
| 5123 |              /* Named parameters not supported in JIT */ |
| 5124 |              TRACE_FRAME_SET_LAST_SEND_UNKNOWN(JIT_G(current_frame)->call); |
| 5125 |              break; |
| 5126 |            } |
| 5127 |            if (!zend_jit_check_func_arg(&dasm_state, opline)) { |
| 5128 |              goto jit_failure; |
| 5129 |            } |
| 5130 |            goto done; |
| 5131 |          case ZEND_CHECK_UNDEF_ARGS: |
| 5132 |            if (JIT_G(current_frame) |
| 5133 |             && JIT_G(current_frame)->call) { |
| 5134 |              TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call); |
| 5135 |            } |
| 5136 |            if (!zend_jit_check_undef_args(&dasm_state, opline)) { |
| 5137 |              goto jit_failure; |
| 5138 |            } |
| 5139 |            goto done; |
| 5140 |          case ZEND_DO_UCALL: |
| 5141 |          case ZEND_DO_ICALL: |
| 5142 |          case ZEND_DO_FCALL_BY_NAME: |
| 5143 |          case ZEND_DO_FCALL: |
| 5144 |            if (!zend_jit_do_fcall(&dasm_state, opline, op_array, op_array_ssa, frame->call_level, -1, p + 1)) { |
| 5145 |              goto jit_failure; |
| 5146 |            } |
| 5147 |            goto done; |
| 5148 |          case ZEND_IS_EQUAL: |
| 5149 |          case ZEND_IS_NOT_EQUAL: |
| 5150 |          case ZEND_IS_SMALLER: |
| 5151 |          case ZEND_IS_SMALLER_OR_EQUAL: |
| 5152 |          case ZEND_CASE: |
| 5153 |            op1_info = OP1_INFO(); |
| 5154 |            op2_info = OP2_INFO(); |
| 5155 |            skip_comparison = |
| 5156 |              ssa_op != ssa->ops && |
| 5157 |              (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG && |
| 5158 |              (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG && |
| 5159 |              zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array); |
| 5160 |            CHECK_OP1_TRACE_TYPE(); |
| 5161 |            CHECK_OP2_TRACE_TYPE(); |
| 5162 |            if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { |
| 5163 |              bool exit_if_true = 0; |
| 5164 |              const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); |
| 5165 |              uint32_t exit_point; |
| 5166 | |
| 5167 |              if (ra) { |
| 5168 |                zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra); |
| 5169 |              } |
| 5170 |              exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); |
| 5171 |              exit_addr = zend_jit_trace_get_exit_addr(exit_point); |
| 5172 |              if (!exit_addr) { |
| 5173 |                goto jit_failure; |
| 5174 |              } |
| 5175 |              smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; |
| 5176 |              if (!zend_jit_cmp(&dasm_state, opline, |
| 5177 |                  op1_info, OP1_RANGE(), OP1_REG_ADDR(), |
| 5178 |                  op2_info, OP2_RANGE(), OP2_REG_ADDR(), |
| 5179 |                  RES_REG_ADDR(), |
| 5180 |                  zend_may_throw(opline, ssa_op, op_array, ssa), |
| 5181 |                  smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) { |
| 5182 |                goto jit_failure; |
| 5183 |              } |
| 5184 |              zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true); |
| 5185 |            } else { |
| 5186 |              smart_branch_opcode = 0; |
| 5187 |              exit_addr = NULL; |
| 5188 |              if (!zend_jit_cmp(&dasm_state, opline, |
| 5189 |                  op1_info, OP1_RANGE(), OP1_REG_ADDR(), |
| 5190 |                  op2_info, OP2_RANGE(), OP2_REG_ADDR(), |
| 5191 |                  RES_REG_ADDR(), |
| 5192 |                  zend_may_throw(opline, ssa_op, op_array, ssa), |
| 5193 |                  smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) { |
| 5194 |                goto jit_failure; |
| 5195 |              } |
| 5196 |            } |
| 5197 |            goto done; |
| 5198 |          case ZEND_IS_IDENTICAL: |
| 5199 |          case ZEND_IS_NOT_IDENTICAL: |
| 5200 |          case ZEND_CASE_STRICT: |
| 5201 |            op1_info = OP1_INFO(); |
| 5202 |            op2_info = OP2_INFO(); |
| 5203 |            skip_comparison = |
| 5204 |              ssa_op != ssa->ops && |
| 5205 |              (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG && |
| 5206 |              (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG && |
| 5207 |              zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array); |
| 5208 |            CHECK_OP1_TRACE_TYPE(); |
| 5209 |            CHECK_OP2_TRACE_TYPE(); |
| 5210 |            if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { |
| 5211 |              bool exit_if_true = 0; |
| 5212 |              const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); |
| 5213 |              uint32_t exit_point; |
| 5214 | |
| 5215 |              if (ra) { |
| 5216 |                zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra); |
| 5217 |              } |
| 5218 |              exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); |
| 5219 |              exit_addr = zend_jit_trace_get_exit_addr(exit_point); |
| 5220 |              if (!exit_addr) { |
| 5221 |                goto jit_failure; |
| 5222 |              } |
| 5223 |              if (opline->opcode == ZEND_IS_NOT_IDENTICAL) { |
| 5224 |                exit_if_true = !exit_if_true; |
| 5225 |              } |
| 5226 |              smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; |
| 5227 |              if (!zend_jit_identical(&dasm_state, opline, |
| 5228 |                  op1_info, OP1_RANGE(), OP1_REG_ADDR(), |
| 5229 |                  op2_info, OP2_RANGE(), OP2_REG_ADDR(), |
| 5230 |                  RES_REG_ADDR(), |
| 5231 |                  zend_may_throw(opline, ssa_op, op_array, ssa), |
| 5232 |                  smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) { |
| 5233 |                goto jit_failure; |
| 5234 |              } |
| 5235 |              zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true); |
| 5236 |            } else { |
| 5237 |              smart_branch_opcode = 0; |
| 5238 |              exit_addr = NULL; |
| 5239 |              if (!zend_jit_identical(&dasm_state, opline, |
| 5240 |                  op1_info, OP1_RANGE(), OP1_REG_ADDR(), |
| 5241 |                  op2_info, OP2_RANGE(), OP2_REG_ADDR(), |
| 5242 |                  RES_REG_ADDR(), |
| 5243 |                  zend_may_throw(opline, ssa_op, op_array, ssa), |
| 5244 |                  smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) { |
| 5245 |                goto jit_failure; |
| 5246 |              } |
| 5247 |            } |
| 5248 |            goto done; |
| 5249 |          case ZEND_DEFINED: |
| 5250 |            if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { |
| 5251 |              bool exit_if_true = 0; |
| 5252 |              const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); |
| 5253 |              uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); |
| 5254 | |
| 5255 |              exit_addr = zend_jit_trace_get_exit_addr(exit_point); |
| 5256 |              if (!exit_addr) { |
| 5257 |                goto jit_failure; |
| 5258 |              } |
| 5259 |              smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; |
| 5260 |            } else { |
| 5261 |              smart_branch_opcode = 0; |
| 5262 |              exit_addr = NULL; |
| 5263 |            } |
| 5264 |            if (!zend_jit_defined(&dasm_state, opline, smart_branch_opcode, -1, -1, exit_addr)) { |
| 5265 |              goto jit_failure; |
| 5266 |            } |
| 5267 |            goto done; |
| 5268 |          case ZEND_TYPE_CHECK: |
| 5269 |            if (opline->extended_value == MAY_BE_RESOURCE) { |
| 5270 |              // TODO: support for is_resource() ??? |
| 5271 |              break; |
| 5272 |            } |
| 5273 |            op1_info = OP1_INFO(); |
| 5274 |            CHECK_OP1_TRACE_TYPE(); |
| 5275 |            if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { |
| 5276 |              bool exit_if_true = 0; |
| 5277 |              const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); |
| 5278 |              uint32_t exit_point; |
| 5279 | |
| 5280 |              if (ra) { |
| 5281 |                zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra); |
| 5282 |              } |
| 5283 |              exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); |
| 5284 |              exit_addr = zend_jit_trace_get_exit_addr(exit_point); |
| 5285 |              if (!exit_addr) { |
| 5286 |                goto jit_failure; |
| 5287 |              } |
| 5288 |              smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; |
| 5289 |            } else { |
| 5290 |              smart_branch_opcode = 0; |
| 5291 |              exit_addr = NULL; |
| 5292 |            } |
| 5293 |            if (!zend_jit_type_check(&dasm_state, opline, op1_info, smart_branch_opcode, -1, -1, exit_addr)) { |
| 5294 |              goto jit_failure; |
| 5295 |            } |
| 5296 |            goto done; |
| 5297 |          case ZEND_RETURN: |
| 5298 |            op1_info = OP1_INFO(); |
| 5299 |            CHECK_OP1_TRACE_TYPE(); |
| 5300 |            if (opline->op1_type == IS_CONST) { |
| 5301 |              res_type = Z_TYPE_P(RT_CONSTANT(opline, opline->op1)); |
| 5302 |            } else if (op1_type != IS_UNKNOWN) { |
| 5303 |              res_type = op1_type; |
| 5304 |            } |
| 5305 |            if ( |
| 5306 |             |
| 5307 |             |
| 5308 |             |
| 5309 |             ) { |
| 5310 |              if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) { |
| 5311 |                goto jit_failure; |
| 5312 |              } |
| 5313 |            } else { |
| 5314 |              int j; |
| 5315 |              int may_throw = 0; |
| 5316 |              bool left_frame = 0; |
| 5317 | |
| 5318 |              if (!zend_jit_return(&dasm_state, opline, op_array, |
| 5319 |                  op1_info, OP1_REG_ADDR())) { |
| 5320 |                goto jit_failure; |
| 5321 |              } |
| 5322 |              if (op_array->last_var > 100) { |
| 5323 |                /* To many CVs to unroll */ |
| 5324 |                if (!zend_jit_free_cvs(&dasm_state)) { |
| 5325 |                  goto jit_failure; |
| 5326 |                } |
| 5327 |                left_frame = 1; |
| 5328 |              } |
| 5329 |              if (!left_frame) { |
| 5330 |                for (j = 0 ; j < op_array->last_var; j++) { |
| 5331 |                  uint32_t info; |
| 5332 |                  zend_uchar type; |
| 5333 | |
| 5334 |                  info = zend_ssa_cv_info(op_array, op_array_ssa, j); |
| 5335 |                  type = STACK_TYPE(stack, j); |
| 5336 |                  info = zend_jit_trace_type_to_info_ex(type, info); |
| 5337 |                  if (opline->op1_type == IS_CV |
| 5338 |                   && EX_VAR_TO_NUM(opline->op1.var) == j |
| 5339 |                   && !(op1_info & (MAY_BE_REF|MAY_BE_OBJECT))) { |
| 5340 |                    if (JIT_G(current_frame) |
| 5341 |                     && TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { |
| 5342 |                      continue; |
| 5343 |                    } else { |
| 5344 |                      info |= MAY_BE_NULL; |
| 5345 |                    } |
| 5346 |                  } |
| 5347 |                  if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { |
| 5348 |                    if (!left_frame) { |
| 5349 |                      left_frame = 1; |
| 5350 |                     if (!zend_jit_leave_frame(&dasm_state)) { |
| 5351 |                        goto jit_failure; |
| 5352 |                     } |
| 5353 |                    } |
| 5354 |                    if (!zend_jit_free_cv(&dasm_state, info, j)) { |
| 5355 |                      goto jit_failure; |
| 5356 |                    } |
| 5357 |                    if (info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_RESOURCE)) { |
| 5358 |                      if (info & MAY_BE_RC1) { |
| 5359 |                        may_throw = 1; |
| 5360 |                      } |
| 5361 |                    } |
| 5362 |                  } |
| 5363 |                } |
| 5364 |              } |
| 5365 |              if (!zend_jit_leave_func(&dasm_state, op_array, opline, op1_info, left_frame, |
| 5366 |                  p + 1, &zend_jit_traces[ZEND_JIT_TRACE_NUM], |
| 5367 |                  (op_array_ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, may_throw)) { |
| 5368 |                goto jit_failure; |
| 5369 |              } |
| 5370 |            } |
| 5371 |            goto done; |
| 5372 |          case ZEND_BOOL: |
| 5373 |          case ZEND_BOOL_NOT: |
| 5374 |            op1_info = OP1_INFO(); |
| 5375 |            CHECK_OP1_TRACE_TYPE(); |
| 5376 |            if (!zend_jit_bool_jmpznz(&dasm_state, opline, |
| 5377 |                op1_info, OP1_REG_ADDR(), RES_REG_ADDR(), |
| 5378 |                -1, -1, |
| 5379 |                zend_may_throw(opline, ssa_op, op_array, ssa), |
| 5380 |                opline->opcode, NULL)) { |
| 5381 |              goto jit_failure; |
| 5382 |            } |
| 5383 |            goto done; |
| 5384 |          case ZEND_JMPZ: |
| 5385 |          case ZEND_JMPNZ: |
| 5386 |          case ZEND_JMPZ_EX: |
| 5387 |          case ZEND_JMPNZ_EX: |
| 5388 |            op1_info = OP1_INFO(); |
| 5389 |            CHECK_OP1_TRACE_TYPE(); |
| 5390 |            if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) { |
| 5391 |              const zend_op *exit_opline = NULL; |
| 5392 |              uint32_t exit_point; |
| 5393 | |
| 5394 |              if ( ) { |
| 5395 |                /* taken branch */ |
| 5396 |                if (opline->opcode == ZEND_JMPNZ_EX) { |
| 5397 |                  smart_branch_opcode = ZEND_JMPZ_EX; |
| 5398 |                } else if (opline->opcode == ZEND_JMPZ_EX) { |
| 5399 |                  smart_branch_opcode = ZEND_JMPNZ_EX; |
| 5400 |                } else if (opline->opcode == ZEND_JMPNZ) { |
| 5401 |                  smart_branch_opcode = ZEND_JMPZ; |
| 5402 |                } else { |
| 5403 |                  smart_branch_opcode = ZEND_JMPNZ; |
| 5404 |                } |
| 5405 |                exit_opline = opline + 1; |
| 5406 |              } else if ( ) { |
| 5407 |                /* not taken branch */ |
| 5408 |                smart_branch_opcode = opline->opcode; |
| 5409 |                exit_opline = OP_JMP_ADDR(opline, opline->op2); |
| 5410 |              } else { |
| 5411 |                ZEND_UNREACHABLE(); |
| 5412 |              } |
| 5413 |              if (ra) { |
| 5414 |                zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra); |
| 5415 |              } |
| 5416 |              if ( |
| 5417 |               |
| 5418 |               ) { |
| 5419 |                /* unconditional branch */ |
| 5420 |                exit_addr = NULL; |
| 5421 |              } else if (opline->result_type == IS_TMP_VAR) { |
| 5422 |                zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; |
| 5423 |                uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); |
| 5424 | |
| 5425 |                SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); |
| 5426 |                exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); |
| 5427 |                SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); |
| 5428 |                exit_addr = zend_jit_trace_get_exit_addr(exit_point); |
| 5429 |                if (!exit_addr) { |
| 5430 |                  goto jit_failure; |
| 5431 |                } |
| 5432 |              } else { |
| 5433 |                exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); |
| 5434 |                exit_addr = zend_jit_trace_get_exit_addr(exit_point); |
| 5435 |                if (!exit_addr) { |
| 5436 |                  goto jit_failure; |
| 5437 |                } |
| 5438 |              } |
| 5439 |            } else { |
| 5440 |              ZEND_UNREACHABLE(); |
| 5441 |            } |
| 5442 |            if (opline->result_type == IS_UNDEF) { |
| 5443 |              res_addr = 0; |
| 5444 |            } else { |
| 5445 |              res_addr = RES_REG_ADDR(); |
| 5446 |            } |
| 5447 |            if (!zend_jit_bool_jmpznz(&dasm_state, opline, |
| 5448 |                op1_info, OP1_REG_ADDR(), res_addr, |
| 5449 |                -1, -1, |
| 5450 |                zend_may_throw(opline, ssa_op, op_array, ssa), |
| 5451 |                smart_branch_opcode, exit_addr)) { |
| 5452 |              goto jit_failure; |
| 5453 |            } |
| 5454 |            goto done; |
| 5455 |          case ZEND_ISSET_ISEMPTY_CV: |
| 5456 |            if ((opline->extended_value & ZEND_ISEMPTY)) { |
| 5457 |              // TODO: support for empty() ??? |
| 5458 |              break; |
| 5459 |            } |
| 5460 |            op1_info = OP1_INFO(); |
| 5461 |            op1_addr = OP1_REG_ADDR(); |
| 5462 |            if (orig_op1_type != IS_UNKNOWN |
| 5463 |             && (orig_op1_type & IS_TRACE_REFERENCE)) { |
| 5464 |              if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, |
| 5465 |                  !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) { |
| 5466 |                goto jit_failure; |
| 5467 |              } |
| 5468 |              if (opline->op1_type == IS_CV |
| 5469 |               && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { |
| 5470 |                ssa->var_info[ssa_op->op1_use].guarded_reference = 1; |
| 5471 |              } |
| 5472 |            } else { |
| 5473 |              CHECK_OP1_TRACE_TYPE(); |
| 5474 |            } |
| 5475 |            if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { |
| 5476 |              bool exit_if_true = 0; |
| 5477 |              const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); |
| 5478 |              uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); |
| 5479 | |
| 5480 |              exit_addr = zend_jit_trace_get_exit_addr(exit_point); |
| 5481 |              if (!exit_addr) { |
| 5482 |                goto jit_failure; |
| 5483 |              } |
| 5484 |              smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; |
| 5485 |            } else { |
| 5486 |              smart_branch_opcode = 0; |
| 5487 |              exit_addr = NULL; |
| 5488 |            } |
| 5489 |            if (!zend_jit_isset_isempty_cv(&dasm_state, opline, |
| 5490 |                op1_info, op1_addr, |
| 5491 |                smart_branch_opcode, -1, -1, exit_addr)) { |
| 5492 |              goto jit_failure; |
| 5493 |            } |
| 5494 |            goto done; |
| 5495 |          case ZEND_IN_ARRAY: |
| 5496 |            if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) { |
| 5497 |              break; |
| 5498 |            } |
| 5499 |            op1_info = OP1_INFO(); |
| 5500 |            op1_addr = OP1_REG_ADDR(); |
| 5501 |            CHECK_OP1_TRACE_TYPE(); |
| 5502 |            if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) { |
| 5503 |              break; |
| 5504 |            } |
| 5505 |            if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { |
| 5506 |              bool exit_if_true = 0; |
| 5507 |              const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); |
| 5508 |              uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); |
| 5509 | |
| 5510 |              exit_addr = zend_jit_trace_get_exit_addr(exit_point); |
| 5511 |              if (!exit_addr) { |
| 5512 |                goto jit_failure; |
| 5513 |              } |
| 5514 |              smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; |
| 5515 |            } else { |
| 5516 |              smart_branch_opcode = 0; |
| 5517 |              exit_addr = NULL; |
| 5518 |            } |
| 5519 |            if (!zend_jit_in_array(&dasm_state, opline, |
| 5520 |                op1_info, op1_addr, |
| 5521 |                smart_branch_opcode, -1, -1, exit_addr)) { |
| 5522 |              goto jit_failure; |
| 5523 |            } |
| 5524 |            goto done; |
| 5525 |          case ZEND_FETCH_DIM_FUNC_ARG: |
| 5526 |            if (!JIT_G(current_frame) |
| 5527 |             || !JIT_G(current_frame)->call |
| 5528 |             || !JIT_G(current_frame)->call->func |
| 5529 |             || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { |
| 5530 |              break; |
| 5531 |            } |
| 5532 |            ZEND_FALLTHROUGH; |
| 5533 |          case ZEND_FETCH_DIM_R: |
| 5534 |          case ZEND_FETCH_DIM_IS: |
| 5535 |          case ZEND_FETCH_LIST_R: |
| 5536 |            op1_info = OP1_INFO(); |
| 5537 |            op1_addr = OP1_REG_ADDR(); |
| 5538 |            if (orig_op1_type != IS_UNKNOWN |
| 5539 |             && (orig_op1_type & IS_TRACE_REFERENCE)) { |
| 5540 |              if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, |
| 5541 |                  !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) { |
| 5542 |                goto jit_failure; |
| 5543 |              } |
| 5544 |              if (opline->op1_type == IS_CV |
| 5545 |               && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { |
| 5546 |                ssa->var_info[ssa_op->op1_use].guarded_reference = 1; |
| 5547 |                if (ssa_op->op1_def >= 0) { |
| 5548 |                  ssa->var_info[ssa_op->op1_def].guarded_reference = 1; |
| 5549 |                } |
| 5550 |              } |
| 5551 |            } else { |
| 5552 |              CHECK_OP1_TRACE_TYPE(); |
| 5553 |            } |
| 5554 |            op2_info = OP2_INFO(); |
| 5555 |            CHECK_OP2_TRACE_TYPE(); |
| 5556 |            res_info = RES_INFO(); |
| 5557 |            avoid_refcounting = |
| 5558 |              ssa_op->op1_use >= 0 && |
| 5559 |              ssa->var_info[ssa_op->op1_use].avoid_refcounting; |
| 5560 |            if (op1_info & MAY_BE_PACKED_GUARD) { |
| 5561 |              ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD; |
| 5562 |            } else if ( |
| 5563 |                 |
| 5564 |                 |
| 5565 |                 |
| 5566 |                 ) { |
| 5567 |              op1_info |= MAY_BE_PACKED_GUARD; |
| 5568 |              if (orig_op1_type & IS_TRACE_PACKED) { |
| 5569 |                op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH); |
| 5570 |                if (op1_type != IS_UNKNOWN) { |
| 5571 |                  ssa->var_info[ssa_op->op1_use].type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH); |
| 5572 |                } |
| 5573 |              } else { |
| 5574 |                op1_info &= ~MAY_BE_ARRAY_PACKED; |
| 5575 |                if (op1_type != IS_UNKNOWN) { |
| 5576 |                  ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_ARRAY_PACKED; |
| 5577 |                } |
| 5578 |              } |
| 5579 |            } |
| 5580 |            if (!zend_jit_fetch_dim_read(&dasm_state, opline, ssa, ssa_op, |
| 5581 |                op1_info, op1_addr, avoid_refcounting, |
| 5582 |                op2_info, res_info, RES_REG_ADDR(), val_type)) { |
| 5583 |              goto jit_failure; |
| 5584 |            } |
| 5585 |            if (ssa_op->op1_def >= 0 && op1_type != IS_UNKNOWN) { |
| 5586 |              ssa->var_info[ssa_op->op1_def].type = ssa->var_info[ssa_op->op1_use].type; |
| 5587 |            } |
| 5588 |            goto done; |
| 5589 |          case ZEND_FETCH_DIM_W: |
| 5590 |          case ZEND_FETCH_DIM_RW: |
| 5591 | //          case ZEND_FETCH_DIM_UNSET: |
| 5592 |          case ZEND_FETCH_LIST_W: |
| 5593 |            if (opline->op1_type != IS_CV |
| 5594 |             && (orig_op1_type == IS_UNKNOWN |
| 5595 |             || !(orig_op1_type & IS_TRACE_INDIRECT))) { |
| 5596 |              break; |
| 5597 |            } |
| 5598 |            op1_info = OP1_INFO(); |
| 5599 |            op1_addr = OP1_REG_ADDR(); |
| 5600 |            if (opline->op1_type == IS_VAR) { |
| 5601 |              if (orig_op1_type != IS_UNKNOWN |
| 5602 |               && (orig_op1_type & IS_TRACE_INDIRECT)) { |
| 5603 |                if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type, |
| 5604 |                    &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) { |
| 5605 |                  goto jit_failure; |
| 5606 |                } |
| 5607 |              } else { |
| 5608 |                break; |
| 5609 |              } |
| 5610 |            } |
| 5611 |            if (orig_op1_type != IS_UNKNOWN |
| 5612 |             && (orig_op1_type & IS_TRACE_REFERENCE)) { |
| 5613 |              if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, |
| 5614 |                  !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) { |
| 5615 |                goto jit_failure; |
| 5616 |              } |
| 5617 |              if (opline->op1_type == IS_CV |
| 5618 |               && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { |
| 5619 |                ssa->var_info[ssa_op->op1_def].guarded_reference = 1; |
| 5620 |              } |
| 5621 |            } else { |
| 5622 |              CHECK_OP1_TRACE_TYPE(); |
| 5623 |            } |
| 5624 |            op2_info = OP2_INFO(); |
| 5625 |            CHECK_OP2_TRACE_TYPE(); |
| 5626 |            op1_def_info = OP1_DEF_INFO(); |
| 5627 |            if (!zend_jit_fetch_dim(&dasm_state, opline, |
| 5628 |                op1_info, op1_addr, op2_info, RES_REG_ADDR(), val_type)) { |
| 5629 |              goto jit_failure; |
| 5630 |            } |
| 5631 |            if (ssa_op->result_def > 0 |
| 5632 |             && (opline->opcode == ZEND_FETCH_DIM_W || opline->opcode == ZEND_FETCH_LIST_W) |
| 5633 |             && !(op1_info & (MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) |
| 5634 |             && !(op2_info & (MAY_BE_UNDEF|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { |
| 5635 |              ssa->var_info[ssa_op->result_def].indirect_reference = 1; |
| 5636 |            } |
| 5637 |            goto done; |
| 5638 |          case ZEND_ISSET_ISEMPTY_DIM_OBJ: |
| 5639 |            if ((opline->extended_value & ZEND_ISEMPTY)) { |
| 5640 |              // TODO: support for empty() ??? |
| 5641 |              break; |
| 5642 |            } |
| 5643 |            op1_info = OP1_INFO(); |
| 5644 |            op1_addr = OP1_REG_ADDR(); |
| 5645 |            if (orig_op1_type != IS_UNKNOWN |
| 5646 |             && (orig_op1_type & IS_TRACE_REFERENCE)) { |
| 5647 |              if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, |
| 5648 |                  !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) { |
| 5649 |                goto jit_failure; |
| 5650 |              } |
| 5651 |              if (opline->op1_type == IS_CV |
| 5652 |               && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { |
| 5653 |                ssa->var_info[ssa_op->op1_use].guarded_reference = 1; |
| 5654 |              } |
| 5655 |            } else { |
| 5656 |              CHECK_OP1_TRACE_TYPE(); |
| 5657 |            } |
| 5658 |            op2_info = OP2_INFO(); |
| 5659 |            CHECK_OP2_TRACE_TYPE(); |
| 5660 |            if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { |
| 5661 |              bool exit_if_true = 0; |
| 5662 |              const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true); |
| 5663 |              uint32_t exit_point; |
| 5664 | |
| 5665 |              if (ra) { |
| 5666 |                zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra); |
| 5667 |              } |
| 5668 |              if (ssa_op->op1_use >= 0 |
| 5669 |               && ssa->var_info[ssa_op->op1_use].avoid_refcounting) { |
| 5670 |                /* Temporary reset ZREG_ZVAL_TRY_ADDREF */ |
| 5671 |                zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; |
| 5672 |                uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); |
| 5673 | |
| 5674 |                SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); |
| 5675 |                exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); |
| 5676 |                SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info); |
| 5677 |              } else { |
| 5678 |                exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); |
| 5679 |              } |
| 5680 |              exit_addr = zend_jit_trace_get_exit_addr(exit_point); |
| 5681 |              if (!exit_addr) { |
| 5682 |                goto jit_failure; |
| 5683 |              } |
| 5684 |              smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ; |
| 5685 |            } else { |
| 5686 |              smart_branch_opcode = 0; |
| 5687 |              exit_addr = NULL; |
| 5688 |            } |
| 5689 |            avoid_refcounting = |
| 5690 |              ssa_op->op1_use >= 0 && |
| 5691 |              ssa->var_info[ssa_op->op1_use].avoid_refcounting; |
| 5692 |            if (op1_info & MAY_BE_PACKED_GUARD) { |
| 5693 |              ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD; |
| 5694 |            } else if ( |
| 5695 |                 |
| 5696 |                 |
| 5697 |                 |
| 5698 |                 ) { |
| 5699 |              op1_info |= MAY_BE_PACKED_GUARD; |
| 5700 |              if (orig_op1_type & IS_TRACE_PACKED) { |
| 5701 |                op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH); |
| 5702 |              } else { |
| 5703 |                op1_info &= ~MAY_BE_ARRAY_PACKED; |
| 5704 |              } |
| 5705 |            } |
| 5706 |            if (!zend_jit_isset_isempty_dim(&dasm_state, opline, |
| 5707 |                op1_info, op1_addr, avoid_refcounting, |
| 5708 |                op2_info, val_type, |
| 5709 |                zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info), |
| 5710 |                smart_branch_opcode, -1, -1, |
| 5711 |                exit_addr)) { |
| 5712 |              goto jit_failure; |
| 5713 |            } |
| 5714 |            goto done; |
| 5715 |          case ZEND_FETCH_OBJ_FUNC_ARG: |
| 5716 |            if (!JIT_G(current_frame) |
| 5717 |             || !JIT_G(current_frame)->call |
| 5718 |             || !JIT_G(current_frame)->call->func |
| 5719 |             || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { |
| 5720 |              break; |
| 5721 |            } |
| 5722 |            ZEND_FALLTHROUGH; |
| 5723 |          case ZEND_FETCH_OBJ_R: |
| 5724 |          case ZEND_FETCH_OBJ_IS: |
| 5725 |          case ZEND_FETCH_OBJ_W: |
| 5726 |            on_this = delayed_fetch_this = 0; |
| 5727 |            avoid_refcounting = 0; |
| 5728 |            if (opline->op2_type != IS_CONST |
| 5729 |             || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING |
| 5730 |             || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') { |
| 5731 |              break; |
| 5732 |            } |
| 5733 |            ce = NULL; |
| 5734 |            ce_is_instanceof = 0; |
| 5735 |            op1_indirect = 0; |
| 5736 |            if (opline->op1_type == IS_UNUSED) { |
| 5737 |              op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; |
| 5738 |              ce = op_array->scope; |
| 5739 |              ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; |
| 5740 |              op1_addr = 0; |
| 5741 |              on_this = 1; |
| 5742 |            } else { |
| 5743 |              op1_info = OP1_INFO(); |
| 5744 |              if (!(op1_info & MAY_BE_OBJECT)) { |
| 5745 |                break; |
| 5746 |              } |
| 5747 |              op1_addr = OP1_REG_ADDR(); |
| 5748 |              if (opline->op1_type == IS_VAR |
| 5749 |               && opline->opcode == ZEND_FETCH_OBJ_W) { |
| 5750 |                if (orig_op1_type != IS_UNKNOWN |
| 5751 |                 && (orig_op1_type & IS_TRACE_INDIRECT)) { |
| 5752 |                  op1_indirect = 1; |
| 5753 |                  if (!zend_jit_fetch_indirect_var(&dasm_state, opline, orig_op1_type, |
| 5754 |                      &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) { |
| 5755 |                    goto jit_failure; |
| 5756 |                  } |
| 5757 |                } |
| 5758 |              } |
| 5759 |              if (orig_op1_type != IS_UNKNOWN |
| 5760 |               && (orig_op1_type & IS_TRACE_REFERENCE)) { |
| 5761 |                if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, |
| 5762 |                    !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) { |
| 5763 |                  goto jit_failure; |
| 5764 |                } |
| 5765 |                if (opline->op1_type == IS_CV |
| 5766 |                 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { |
| 5767 |                  ssa->var_info[ssa_op->op1_def >= 0 ? ssa_op->op1_def : ssa_op->op1_use].guarded_reference = 1; |
| 5768 |                } |
| 5769 |              } else { |
| 5770 |                CHECK_OP1_TRACE_TYPE(); |
| 5771 |              } |
| 5772 |              if (!(op1_info & MAY_BE_OBJECT)) { |
| 5773 |                break; |
| 5774 |              } |
| 5775 |              if (ssa->var_info && ssa->ops) { |
| 5776 |                if (ssa_op->op1_use >= 0) { |
| 5777 |                  zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use; |
| 5778 |                  if (op1_ssa->ce && !op1_ssa->ce->create_object) { |
| 5779 |                    ce = op1_ssa->ce; |
| 5780 |                    ce_is_instanceof = op1_ssa->is_instanceof; |
| 5781 |                  } |
| 5782 |                } |
| 5783 |              } |
| 5784 |              if (ssa_op->op1_use >= 0) { |
| 5785 |                delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this; |
| 5786 |                avoid_refcounting = ssa->var_info[ssa_op->op1_use].avoid_refcounting; |
| 5787 |              } |
| 5788 |              if (delayed_fetch_this) { |
| 5789 |                on_this = 1; |
| 5790 |              } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) { |
| 5791 |                on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS; |
| 5792 |              } else if (op_array_ssa->ops |
| 5793 |               && op_array_ssa->vars |
| 5794 |                  && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0 |
| 5795 |                  && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) { |
| 5796 |                on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS; |
| 5797 |              } |
| 5798 |            } |
| 5799 |            if (!zend_jit_fetch_obj(&dasm_state, opline, op_array, ssa, ssa_op, |
| 5800 |                op1_info, op1_addr, op1_indirect, ce, ce_is_instanceof, |
| 5801 |                on_this, delayed_fetch_this, avoid_refcounting, op1_ce, val_type, |
| 5802 |                zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, MAY_BE_STRING))) { |
| 5803 |              goto jit_failure; |
| 5804 |            } |
| 5805 |            goto done; |
| 5806 |          case ZEND_BIND_GLOBAL: |
| 5807 |            orig_opline = opline; |
| 5808 |            orig_ssa_op = ssa_op; |
| 5809 |            while (1) { |
| 5810 |              if (!ssa->ops || !ssa->var_info) { |
| 5811 |                op1_info = MAY_BE_ANY|MAY_BE_REF; |
| 5812 |              } else { |
| 5813 |                op1_info = OP1_INFO(); |
| 5814 |              } |
| 5815 |              if (ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) { |
| 5816 |                ssa->var_info[ssa_op->op1_def].guarded_reference = 1; |
| 5817 |              } |
| 5818 |              if (!zend_jit_bind_global(&dasm_state, opline, op1_info)) { |
| 5819 |                goto jit_failure; |
| 5820 |              } |
| 5821 |              if ((opline+1)->opcode == ZEND_BIND_GLOBAL) { |
| 5822 |                opline++; |
| 5823 |                ssa_op++; |
| 5824 |              } else { |
| 5825 |                break; |
| 5826 |              } |
| 5827 |            } |
| 5828 |            opline = orig_opline; |
| 5829 |            ssa_op = orig_ssa_op; |
| 5830 |            goto done; |
| 5831 |          case ZEND_RECV: |
| 5832 |            if (!zend_jit_recv(&dasm_state, opline, op_array)) { |
| 5833 |              goto jit_failure; |
| 5834 |            } |
| 5835 |            goto done; |
| 5836 |          case ZEND_RECV_INIT: |
| 5837 |            orig_opline = opline; |
| 5838 |            orig_ssa_op = ssa_op; |
| 5839 |            while (1) { |
| 5840 |              if (!zend_jit_recv_init(&dasm_state, opline, op_array, |
| 5841 |                  (opline + 1)->opcode != ZEND_RECV_INIT, |
| 5842 |                  zend_may_throw(opline, ssa_op, op_array, ssa))) { |
| 5843 |                goto jit_failure; |
| 5844 |              } |
| 5845 |              if ((opline+1)->opcode == ZEND_RECV_INIT) { |
| 5846 |                opline++; |
| 5847 |                ssa_op++; |
| 5848 |              } else { |
| 5849 |                break; |
| 5850 |              } |
| 5851 |            } |
| 5852 |            opline = orig_opline; |
| 5853 |            ssa_op = orig_ssa_op; |
| 5854 |            goto done; |
| 5855 |          case ZEND_FREE: |
| 5856 |          case ZEND_FE_FREE: |
| 5857 |            op1_info = OP1_INFO(); |
| 5858 |            if (!zend_jit_free(&dasm_state, opline, op1_info, |
| 5859 |                zend_may_throw(opline, ssa_op, op_array, ssa))) { |
| 5860 |              goto jit_failure; |
| 5861 |            } |
| 5862 |            goto done; |
| 5863 |          case ZEND_ECHO: |
| 5864 |            op1_info = OP1_INFO(); |
| 5865 |            CHECK_OP1_TRACE_TYPE(); |
| 5866 |            if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) { |
| 5867 |              break; |
| 5868 |            } |
| 5869 |            if (!zend_jit_echo(&dasm_state, opline, op1_info)) { |
| 5870 |              goto jit_failure; |
| 5871 |            } |
| 5872 |            goto done; |
| 5873 |          case ZEND_STRLEN: |
| 5874 |            op1_info = OP1_INFO(); |
| 5875 |            op1_addr = OP1_REG_ADDR(); |
| 5876 |            if (orig_op1_type == (IS_TRACE_REFERENCE|IS_STRING)) { |
| 5877 |              if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, |
| 5878 |                  !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) { |
| 5879 |                goto jit_failure; |
| 5880 |              } |
| 5881 |              if (opline->op1_type == IS_CV |
| 5882 |               && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { |
| 5883 |                ssa->var_info[ssa_op->op1_use].guarded_reference = 1; |
| 5884 |              } |
| 5885 |            } else { |
| 5886 |              CHECK_OP1_TRACE_TYPE(); |
| 5887 |              if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) { |
| 5888 |                break; |
| 5889 |              } |
| 5890 |            } |
| 5891 |            if (!zend_jit_strlen(&dasm_state, opline, op1_info, op1_addr, RES_REG_ADDR())) { |
| 5892 |              goto jit_failure; |
| 5893 |            } |
| 5894 |            goto done; |
| 5895 |          case ZEND_COUNT: |
| 5896 |            op1_info = OP1_INFO(); |
| 5897 |            op1_addr = OP1_REG_ADDR(); |
| 5898 |            if (orig_op1_type == (IS_TRACE_REFERENCE|IS_ARRAY)) { |
| 5899 |              if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, |
| 5900 |                  !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) { |
| 5901 |                goto jit_failure; |
| 5902 |              } |
| 5903 |              if (opline->op1_type == IS_CV |
| 5904 |               && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { |
| 5905 |                ssa->var_info[ssa_op->op1_use].guarded_reference = 1; |
| 5906 |              } |
| 5907 |            } else { |
| 5908 |              CHECK_OP1_TRACE_TYPE(); |
| 5909 |              if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) { |
| 5910 |                break; |
| 5911 |              } |
| 5912 |            } |
| 5913 |            if (!zend_jit_count(&dasm_state, opline, op1_info, op1_addr, RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) { |
| 5914 |              goto jit_failure; |
| 5915 |            } |
| 5916 |            goto done; |
| 5917 |          case ZEND_FETCH_THIS: |
| 5918 |            delayed_fetch_this = 0; |
| 5919 |            if (ssa_op->result_def >= 0 && opline->result_type != IS_CV) { |
| 5920 |              if (zend_jit_may_delay_fetch_this(op_array, ssa, ssa_opcodes, ssa_op)) { |
| 5921 |                ssa->var_info[ssa_op->result_def].delayed_fetch_this = 1; |
| 5922 |                delayed_fetch_this = 1; |
| 5923 |              } |
| 5924 |            } |
| 5925 |            if (!zend_jit_fetch_this(&dasm_state, opline, op_array, delayed_fetch_this)) { |
| 5926 |              goto jit_failure; |
| 5927 |            } |
| 5928 |            goto done; |
| 5929 |          case ZEND_SWITCH_LONG: |
| 5930 |          case ZEND_SWITCH_STRING: |
| 5931 |          case ZEND_MATCH: |
| 5932 |            if (!zend_jit_switch(&dasm_state, opline, op_array, op_array_ssa, p+1, &zend_jit_traces[ZEND_JIT_TRACE_NUM])) { |
| 5933 |              goto jit_failure; |
| 5934 |            } |
| 5935 |            goto done; |
| 5936 |          case ZEND_VERIFY_RETURN_TYPE: |
| 5937 |            if (opline->op1_type == IS_UNUSED) { |
| 5938 |              /* Always throws */ |
| 5939 |              break; |
| 5940 |            } |
| 5941 |            if (opline->op1_type == IS_CONST) { |
| 5942 |              /* TODO Different instruction format, has return value */ |
| 5943 |              break; |
| 5944 |            } |
| 5945 |            if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) { |
| 5946 |              /* Not worth bothering with */ |
| 5947 |              break; |
| 5948 |            } |
| 5949 |            op1_info = OP1_INFO(); |
| 5950 |            CHECK_OP1_TRACE_TYPE(); |
| 5951 |            if (op1_info & MAY_BE_REF) { |
| 5952 |              /* TODO May need reference unwrapping. */ |
| 5953 |              break; |
| 5954 |            } |
| 5955 |            if (!zend_jit_verify_return_type(&dasm_state, opline, op_array, op1_info)) { |
| 5956 |              goto jit_failure; |
| 5957 |            } |
| 5958 |            goto done; |
| 5959 |          case ZEND_FE_RESET_R: |
| 5960 |            op1_info = OP1_INFO(); |
| 5961 |            CHECK_OP1_TRACE_TYPE(); |
| 5962 |            if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) { |
| 5963 |              break; |
| 5964 |            } |
| 5965 |            if (!zend_jit_fe_reset(&dasm_state, opline, op1_info)) { |
| 5966 |              goto jit_failure; |
| 5967 |            } |
| 5968 |            goto done; |
| 5969 |          case ZEND_FE_FETCH_R: |
| 5970 |            op1_info = OP1_INFO(); |
| 5971 |            CHECK_OP1_TRACE_TYPE(); |
| 5972 |            if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) { |
| 5973 |              break; |
| 5974 |            } |
| 5975 |            if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) { |
| 5976 |              const zend_op *exit_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); |
| 5977 |              uint32_t exit_point; |
| 5978 | |
| 5979 |              if ((p+1)->opline == exit_opline) { |
| 5980 |                /* taken branch (exit from loop) */ |
| 5981 |                exit_opline = opline; |
| 5982 |                smart_branch_opcode = ZEND_NOP; |
| 5983 |              } else if ( ) { |
| 5984 |                /* not taken branch (loop) */ |
| 5985 |                smart_branch_opcode = ZEND_JMP; |
| 5986 |              } else { |
| 5987 |                ZEND_UNREACHABLE(); |
| 5988 |              } |
| 5989 |              exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); |
| 5990 |              exit_addr = zend_jit_trace_get_exit_addr(exit_point); |
| 5991 |              if (!exit_addr) { |
| 5992 |                goto jit_failure; |
| 5993 |              } |
| 5994 |            } else { |
| 5995 |              ZEND_UNREACHABLE(); |
| 5996 |            } |
| 5997 |            if (!zend_jit_fe_fetch(&dasm_state, opline, op1_info, OP2_INFO(), |
| 5998 |                -1, smart_branch_opcode, exit_addr)) { |
| 5999 |              goto jit_failure; |
| 6000 |            } |
| 6001 |            goto done; |
| 6002 |          case ZEND_FETCH_CONSTANT: |
| 6003 |            if (!zend_jit_fetch_constant(&dasm_state, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) { |
| 6004 |              goto jit_failure; |
| 6005 |            } |
| 6006 |            goto done; |
| 6007 |          case ZEND_INIT_METHOD_CALL: |
| 6008 |            if (opline->op2_type != IS_CONST |
| 6009 |             || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) { |
| 6010 |              goto generic_dynamic_call; |
| 6011 |            } |
| 6012 |            on_this = delayed_fetch_this = 0; |
| 6013 |            ce = NULL; |
| 6014 |            ce_is_instanceof = 0; |
| 6015 |            if (opline->op1_type == IS_UNUSED) { |
| 6016 |              op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN; |
| 6017 |              ce = op_array->scope; |
| 6018 |              ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0; |
| 6019 |              op1_addr = 0; |
| 6020 |              on_this = 1; |
| 6021 |            } else { |
| 6022 |              op1_info = OP1_INFO(); |
| 6023 |              op1_addr = OP1_REG_ADDR(); |
| 6024 |              if (polymorphic_side_trace) { |
| 6025 |                op1_info = MAY_BE_OBJECT; |
| 6026 |                op1_addr = 0; |
| 6027 |              } else if (orig_op1_type != IS_UNKNOWN |
| 6028 |               && (orig_op1_type & IS_TRACE_REFERENCE)) { |
| 6029 |                if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr, |
| 6030 |                    !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) { |
| 6031 |                  goto jit_failure; |
| 6032 |                } |
| 6033 |                if (opline->op1_type == IS_CV |
| 6034 |                 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) { |
| 6035 |                  ssa->var_info[ssa_op->op1_use].guarded_reference = 1; |
| 6036 |                } |
| 6037 |              } else { |
| 6038 |                CHECK_OP1_TRACE_TYPE(); |
| 6039 |              } |
| 6040 |              if (ssa->var_info && ssa->ops) { |
| 6041 |                if (ssa_op->op1_use >= 0) { |
| 6042 |                  zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use; |
| 6043 |                  if (op1_ssa->ce && !op1_ssa->ce->create_object) { |
| 6044 |                    ce = op1_ssa->ce; |
| 6045 |                    ce_is_instanceof = op1_ssa->is_instanceof; |
| 6046 |                  } |
| 6047 |                } |
| 6048 |              } |
| 6049 |              if (ssa_op->op1_use >= 0) { |
| 6050 |                delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this; |
| 6051 |              } |
| 6052 |              if (delayed_fetch_this) { |
| 6053 |                on_this = 1; |
| 6054 |              } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) { |
| 6055 |                on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS; |
| 6056 |              } else if (op_array_ssa->ops |
| 6057 |               && op_array_ssa->vars |
| 6058 |                  && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0 |
| 6059 |                  && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) { |
| 6060 |                on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS; |
| 6061 |              } |
| 6062 |            } |
| 6063 |            frame_flags = TRACE_FRAME_MASK_NESTED; |
| 6064 |            if (!zend_jit_init_method_call(&dasm_state, opline, |
| 6065 |                op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, |
| 6066 |                op_array, ssa, ssa_op, frame->call_level, |
| 6067 |                op1_info, op1_addr, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce, |
| 6068 |                p + 1, peek_checked_stack - checked_stack, polymorphic_side_trace)) { |
| 6069 |              goto jit_failure; |
| 6070 |            } |
| 6071 |            goto done; |
| 6072 |          case ZEND_INIT_DYNAMIC_CALL: |
| 6073 |            if (orig_op2_type != IS_OBJECT || op2_ce != zend_ce_closure) { |
| 6074 |              goto generic_dynamic_call; |
| 6075 |            } |
| 6076 |            op2_info = OP2_INFO(); |
| 6077 |            CHECK_OP2_TRACE_TYPE(); |
| 6078 |            frame_flags = TRACE_FRAME_MASK_NESTED; |
| 6079 |            if (!zend_jit_init_closure_call(&dasm_state, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) { |
| 6080 |              goto jit_failure; |
| 6081 |            } |
| 6082 |            goto done; |
| 6083 |          case ZEND_INIT_STATIC_METHOD_CALL: |
| 6084 | generic_dynamic_call: |
| 6085 |            if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) { |
| 6086 |              goto jit_failure; |
| 6087 |            } |
| 6088 |            if ((p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func |
| 6089 |             && (opline->opcode != ZEND_INIT_STATIC_METHOD_CALL |
| 6090 |             || opline->op1_type != IS_CONST |
| 6091 |             || opline->op2_type != IS_CONST |
| 6092 |             || zend_jit_may_be_modified((p+1)->func, op_array))) { |
| 6093 |              if (!zend_jit_init_fcall_guard(&dasm_state, 0, (p+1)->func, opline+1)) { |
| 6094 |                goto jit_failure; |
| 6095 |              } |
| 6096 |            } |
| 6097 |            goto done; |
| 6098 |          case ZEND_INIT_USER_CALL: |
| 6099 |            if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) { |
| 6100 |              goto jit_failure; |
| 6101 |            } |
| 6102 |            if ((p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func |
| 6103 |             && (opline->op2_type != IS_CONST |
| 6104 |             || zend_jit_may_be_modified((p+1)->func, op_array))) { |
| 6105 |              if (!zend_jit_init_fcall_guard(&dasm_state, 0, (p+1)->func, opline+1)) { |
| 6106 |                goto jit_failure; |
| 6107 |              } |
| 6108 |            } |
| 6109 |            goto done; |
| 6110 |          case ZEND_NEW: |
| 6111 |            if (!zend_jit_trace_handler(&dasm_state, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) { |
| 6112 |              goto jit_failure; |
| 6113 |            } |
| 6114 |            if ((p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func |
| 6115 |             && (opline->op1_type != IS_CONST |
| 6116 |             || zend_jit_may_be_modified((p+1)->func, op_array))) { |
| 6117 |              SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_OBJECT, 1); |
| 6118 |              if (!zend_jit_init_fcall_guard(&dasm_state, 0, (p+1)->func, opline+1)) { |
| 6119 |                goto jit_failure; |
| 6120 |              } |
| 6121 |            } |
| 6122 |            goto done; |
| 6123 |          case ZEND_SEND_ARRAY: |
| 6124 |          case ZEND_SEND_UNPACK: |
| 6125 |            if (JIT_G(current_frame) |
| 6126 |             && JIT_G(current_frame)->call) { |
| 6127 |              TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call); |
| 6128 |            } |
| 6129 |            break; |
| 6130 |          case ZEND_ROPE_INIT: |
| 6131 |          case ZEND_ROPE_ADD: |
| 6132 |          case ZEND_ROPE_END: |
| 6133 |            op2_info = OP2_INFO(); |
| 6134 |            CHECK_OP2_TRACE_TYPE(); |
| 6135 |            if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) { |
| 6136 |              break; |
| 6137 |            } |
| 6138 |            if (!zend_jit_rope(&dasm_state, opline, op2_info)) { |
| 6139 |              goto jit_failure; |
| 6140 |            } |
| 6141 |            goto done; |
| 6142 |          default: |
| 6143 |            break; |
| 6144 |        } |
| 6145 |      } |
| 6146 | |
| 6147 |      if (opline->opcode != ZEND_NOP && opline->opcode != ZEND_JMP) { |
| 6148 |        op1_info = OP1_INFO(); |
| 6149 |        op2_info = OP2_INFO(); |
| 6150 |        if (op1_info & MAY_BE_GUARD) { |
| 6151 |          op1_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; |
| 6152 |        } |
| 6153 |        if (op2_info & MAY_BE_GUARD) { |
| 6154 |          op2_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; |
| 6155 |        } |
| 6156 |        if (!zend_jit_trace_handler(&dasm_state, op_array, opline, |
| 6157 |            zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info), p + 1)) { |
| 6158 |          goto jit_failure; |
| 6159 |        } |
| 6160 |      } |
| 6161 | |
| 6162 | done: |
| 6163 |      polymorphic_side_trace = 0; |
| 6164 |      switch (opline->opcode) { |
| 6165 |        case ZEND_DO_FCALL: |
| 6166 |        case ZEND_DO_ICALL: |
| 6167 |        case ZEND_DO_UCALL: |
| 6168 |        case ZEND_DO_FCALL_BY_NAME: |
| 6169 |          frame->call_level--; |
| 6170 |      } |
| 6171 | |
| 6172 |      if (ra) { |
| 6173 |        zend_jit_trace_clenup_stack(stack, opline, ssa_op, ssa, ra); |
| 6174 |      } |
| 6175 | |
| 6176 |      if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) |
| 6177 |       && STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var)) > ZREG_NUM) { |
| 6178 |        SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); |
| 6179 |      } |
| 6180 | |
| 6181 |      if (opline->opcode == ZEND_ROPE_INIT) { |
| 6182 |        /* clear stack slots used by rope */ |
| 6183 |        uint32_t var = EX_VAR_TO_NUM(opline->result.var); |
| 6184 |        uint32_t count = |
| 6185 |          ((opline->extended_value * sizeof(void*)) + (sizeof(zval)-1)) / sizeof(zval); |
| 6186 | |
| 6187 |        do { |
| 6188 |          SET_STACK_TYPE(stack, var, IS_UNKNOWN, 1); |
| 6189 |          var++; |
| 6190 |          count--; |
| 6191 |        } while (count); |
| 6192 |      } |
| 6193 | |
| 6194 |      if (ssa_op) { |
| 6195 |        zend_ssa_range tmp; |
| 6196 | |
| 6197 |        /* Keep information about known types on abstract stack */ |
| 6198 |        if (ssa_op->result_def >= 0) { |
| 6199 |          zend_uchar type = IS_UNKNOWN; |
| 6200 | |
| 6201 |          if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0 |
| 6202 |           || send_result) { |
| 6203 |            /* we didn't set result variable */ |
| 6204 |            type = IS_UNKNOWN; |
| 6205 |          } else if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD) |
| 6206 |           && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) { |
| 6207 |            type = concrete_type(ssa->var_info[ssa_op->result_def].type); |
| 6208 |          } else if (opline->opcode == ZEND_QM_ASSIGN) { |
| 6209 |            if (opline->op1_type != IS_CONST) { |
| 6210 |              /* copy */ |
| 6211 |              type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)); |
| 6212 |            } |
| 6213 |          } else if (opline->opcode == ZEND_ASSIGN) { |
| 6214 |            if (opline->op2_type != IS_CONST |
| 6215 |             && ssa_op->op1_use >= 0 |
| 6216 |             /* assignment to typed reference may cause conversion */ |
| 6217 |             && (ssa->var_info[ssa_op->op1_use].type & MAY_BE_REF) == 0) { |
| 6218 |              /* copy */ |
| 6219 |              type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var)); |
| 6220 |            } |
| 6221 |          } else if (opline->opcode == ZEND_POST_INC |
| 6222 |       || opline->opcode == ZEND_POST_DEC) { |
| 6223 |            /* copy */ |
| 6224 |            type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)); |
| 6225 |          } |
| 6226 |          if (opline->opcode == ZEND_JMP_SET |
| 6227 |           || opline->opcode == ZEND_COALESCE |
| 6228 |           || opline->opcode == ZEND_JMP_NULL) { |
| 6229 |            if ((p+1)->op != ZEND_JIT_TRACE_VM) { |
| 6230 |              SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); |
| 6231 |            } else if ( ) { |
| 6232 |              SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1); |
| 6233 |            } |
| 6234 |          } else { |
| 6235 |            SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, |
| 6236 |              (type == IS_UNKNOWN || !ra || !ra[ssa_op->result_def])); |
| 6237 |            if (ssa->var_info[ssa_op->result_def].type & MAY_BE_INDIRECT) { |
| 6238 |              RESET_STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)); |
| 6239 |            } |
| 6240 |            if (type != IS_UNKNOWN) { |
| 6241 |              ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; |
| 6242 |              if (opline->opcode == ZEND_FETCH_THIS |
| 6243 |               && delayed_fetch_this) { |
| 6244 |                SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_THIS); |
| 6245 |              } else if (ssa->var_info[ssa_op->result_def].avoid_refcounting) { |
| 6246 |                SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_TRY_ADDREF); |
| 6247 |              } else if (ra && ra[ssa_op->result_def]) { |
| 6248 |                SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def]->reg, |
| 6249 |                  ra[ssa_op->result_def]->flags & ZREG_STORE); |
| 6250 |              } |
| 6251 |            } |
| 6252 |          } |
| 6253 | |
| 6254 |          if (type == IS_LONG |
| 6255 |           && zend_inference_propagate_range(op_array, ssa, (zend_op*)opline, (zend_ssa_op*)ssa_op, ssa_op->result_def, &tmp)) { |
| 6256 |            ssa->var_info[ssa_op->result_def].range.min = tmp.min; |
| 6257 |            ssa->var_info[ssa_op->result_def].range.max = tmp.max; |
| 6258 |            ssa->var_info[ssa_op->result_def].range.underflow = 0; |
| 6259 |            ssa->var_info[ssa_op->result_def].range.overflow = 0; |
| 6260 |            ssa->var_info[ssa_op->result_def].has_range = 1; |
| 6261 |          } |
| 6262 |        } |
| 6263 |        if (ssa_op->op1_def >= 0 |
| 6264 |         && ( |
| 6265 |         |
| 6266 |         )) { |
| 6267 |          zend_uchar type = IS_UNKNOWN; |
| 6268 | |
| 6269 |          if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD) |
| 6270 |           && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) { |
| 6271 |            type = concrete_type(ssa->var_info[ssa_op->op1_def].type); |
| 6272 |          } else if (opline->opcode == ZEND_ASSIGN) { |
| 6273 |            if (!(OP1_INFO() & MAY_BE_REF) |
| 6274 |             || STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) != IS_UNKNOWN) { |
| 6275 |              if (opline->op2_type != IS_CONST) { |
| 6276 |                /* copy */ |
| 6277 |                type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var)); |
| 6278 |              } |
| 6279 |            } |
| 6280 |          } else if (opline->opcode == ZEND_SEND_VAR |
| 6281 |           || opline->opcode == ZEND_CAST |
| 6282 |           || opline->opcode == ZEND_QM_ASSIGN |
| 6283 |           || opline->opcode == ZEND_JMP_SET |
| 6284 |           || opline->opcode == ZEND_COALESCE |
| 6285 |           || opline->opcode == ZEND_JMP_NULL |
| 6286 |           || opline->opcode == ZEND_FE_RESET_R) { |
| 6287 |            /* keep old value */ |
| 6288 |            type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)); |
| 6289 |          } |
| 6290 |          SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type, |
| 6291 |            (type == IS_UNKNOWN || !ra || |
| 6292 |              (!ra[ssa_op->op1_def] && |
| 6293 |                (opline->opcode == ZEND_ASSIGN || !ssa->vars[ssa_op->op1_def].no_val)))); |
| 6294 |          if (type != IS_UNKNOWN) { |
| 6295 |            ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD; |
| 6296 |            if (ra && ra[ssa_op->op1_def]) { |
| 6297 |              uint8_t flags = ra[ssa_op->op1_def]->flags & ZREG_STORE; |
| 6298 | |
| 6299 |              if (ssa_op->op1_use >= 0) { |
| 6300 |                if (opline->opcode == ZEND_SEND_VAR |
| 6301 |                 || opline->opcode == ZEND_CAST |
| 6302 |                 || opline->opcode == ZEND_QM_ASSIGN |
| 6303 |                 || opline->opcode == ZEND_JMP_SET |
| 6304 |                 || opline->opcode == ZEND_COALESCE |
| 6305 |                 || opline->opcode == ZEND_JMP_NULL |
| 6306 |                 || opline->opcode == ZEND_FE_RESET_R) { |
| 6307 |                  if (!ra[ssa_op->op1_use]) { |
| 6308 |                    flags |= ZREG_LOAD; |
| 6309 |                  } |
| 6310 |                } |
| 6311 |              } |
| 6312 |              SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg, flags); |
| 6313 |            } |
| 6314 |          } |
| 6315 |          if (type == IS_LONG |
| 6316 |           && zend_inference_propagate_range(op_array, ssa, (zend_op*)opline, (zend_ssa_op*)ssa_op, ssa_op->op1_def, &tmp)) { |
| 6317 |            ssa->var_info[ssa_op->op1_def].range.min = tmp.min; |
| 6318 |            ssa->var_info[ssa_op->op1_def].range.max = tmp.max; |
| 6319 |            ssa->var_info[ssa_op->op1_def].range.underflow = 0; |
| 6320 |            ssa->var_info[ssa_op->op1_def].range.overflow = 0; |
| 6321 |            ssa->var_info[ssa_op->op1_def].has_range = 1; |
| 6322 |          } |
| 6323 |        } |
| 6324 |        if (ssa_op->op2_def >= 0 |
| 6325 |         && ( |
| 6326 |         |
| 6327 |         )) { |
| 6328 |          zend_uchar type = IS_UNKNOWN; |
| 6329 | |
| 6330 |          if (!(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD) |
| 6331 |           && has_concrete_type(ssa->var_info[ssa_op->op2_def].type)) { |
| 6332 |            type = concrete_type(ssa->var_info[ssa_op->op2_def].type); |
| 6333 |          } else if (opline->opcode == ZEND_ASSIGN) { |
| 6334 |            /* keep old value */ |
| 6335 |            type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var)); |
| 6336 |          } |
| 6337 |          SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), type, |
| 6338 |            (type == IS_UNKNOWN || !ra || |
| 6339 |              (!ra[ssa_op->op2_def] && !ssa->vars[ssa_op->op2_def].no_val))); |
| 6340 |          if (type != IS_UNKNOWN) { |
| 6341 |            ssa->var_info[ssa_op->op2_def].type &= ~MAY_BE_GUARD; |
| 6342 |            if (ra && ra[ssa_op->op2_def]) { |
| 6343 |              uint8_t flags = ra[ssa_op->op2_def]->flags & ZREG_STORE; |
| 6344 | |
| 6345 |              if (ssa_op->op2_use >= 0) { |
| 6346 |                if (opline->opcode == ZEND_ASSIGN) { |
| 6347 |                  if (!ra[ssa_op->op2_use] |
| 6348 |                   || ra[ssa_op->op2_use]->reg != ra[ssa_op->op2_def]->reg) { |
| 6349 |                    flags |= ZREG_LOAD; |
| 6350 |                  } |
| 6351 |                } |
| 6352 |              } |
| 6353 |              SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op2.var), ra[ssa_op->op2_def]->reg, flags); |
| 6354 |            } |
| 6355 |          } |
| 6356 |          if (type == IS_LONG |
| 6357 |           && zend_inference_propagate_range(op_array, ssa, (zend_op*)opline, (zend_ssa_op*)ssa_op, ssa_op->op2_def, &tmp)) { |
| 6358 |            ssa->var_info[ssa_op->op2_def].range.min = tmp.min; |
| 6359 |            ssa->var_info[ssa_op->op2_def].range.max = tmp.max; |
| 6360 |            ssa->var_info[ssa_op->op2_def].range.underflow = 0; |
| 6361 |            ssa->var_info[ssa_op->op2_def].range.overflow = 0; |
| 6362 |            ssa->var_info[ssa_op->op2_def].has_range = 1; |
| 6363 |          } |
| 6364 |        } |
| 6365 | |
| 6366 |        switch (opline->opcode) { |
| 6367 |          case ZEND_ASSIGN_DIM: |
| 6368 |          case ZEND_ASSIGN_OBJ: |
| 6369 |          case ZEND_ASSIGN_STATIC_PROP: |
| 6370 |          case ZEND_ASSIGN_DIM_OP: |
| 6371 |          case ZEND_ASSIGN_OBJ_OP: |
| 6372 |          case ZEND_ASSIGN_STATIC_PROP_OP: |
| 6373 |          case ZEND_ASSIGN_OBJ_REF: |
| 6374 |          case ZEND_ASSIGN_STATIC_PROP_REF: |
| 6375 |            /* OP_DATA */ |
| 6376 |            ssa_op++; |
| 6377 |            opline++; |
| 6378 |            if (ssa_op->op1_def >= 0) { |
| 6379 |              zend_uchar type = IS_UNKNOWN; |
| 6380 | |
| 6381 |              if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD) |
| 6382 |               && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) { |
| 6383 |                type = concrete_type(ssa->var_info[ssa_op->op1_def].type); |
| 6384 |              } else if ((opline-1)->opcode == ZEND_ASSIGN_DIM |
| 6385 |               || (opline-1)->opcode == ZEND_ASSIGN_OBJ |
| 6386 |               || (opline-1)->opcode == ZEND_ASSIGN_STATIC_PROP) { |
| 6387 |                /* keep old value */ |
| 6388 |                type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)); |
| 6389 |              } |
| 6390 |              SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type, |
| 6391 |                (type == IS_UNKNOWN || !ra || !ra[ssa_op->op1_def])); |
| 6392 |              if (type != IS_UNKNOWN) { |
| 6393 |                ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD; |
| 6394 |                if (ra && ra[ssa_op->op1_def]) { |
| 6395 |                  SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg, |
| 6396 |                    ra[ssa_op->op1_def]->reg & ZREG_STORE); |
| 6397 |                } |
| 6398 |              } |
| 6399 |              if (type == IS_LONG |
| 6400 |               && zend_inference_propagate_range(op_array, ssa, (zend_op*)opline, (zend_ssa_op*)ssa_op, ssa_op->op1_def, &tmp)) { |
| 6401 |                ssa->var_info[ssa_op->op1_def].range.min = tmp.min; |
| 6402 |                ssa->var_info[ssa_op->op1_def].range.max = tmp.max; |
| 6403 |                ssa->var_info[ssa_op->op1_def].range.underflow = 0; |
| 6404 |                ssa->var_info[ssa_op->op1_def].range.overflow = 0; |
| 6405 |                ssa->var_info[ssa_op->op1_def].has_range = 1; |
| 6406 |              } |
| 6407 |            } |
| 6408 |            ssa_op++; |
| 6409 |            break; |
| 6410 |          case ZEND_RECV_INIT: |
| 6411 |           ssa_op++; |
| 6412 |            opline++; |
| 6413 |            while (opline->opcode == ZEND_RECV_INIT) { |
| 6414 |              if (ssa_op->result_def >= 0) { |
| 6415 |                zend_uchar type = IS_UNKNOWN; |
| 6416 | |
| 6417 |                if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD) |
| 6418 |                 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) { |
| 6419 |                  type = concrete_type(ssa->var_info[ssa_op->result_def].type); |
| 6420 |                } |
| 6421 |                SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, |
| 6422 |                  (!ra || !ra[ssa_op->result_def])); |
| 6423 |                if (ra && ra[ssa_op->result_def]) { |
| 6424 |                  SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def]->reg, |
| 6425 |                    ra[ssa_op->result_def]->flags & ZREG_STORE); |
| 6426 |                } |
| 6427 |              } |
| 6428 |              ssa_op++; |
| 6429 |              opline++; |
| 6430 |            } |
| 6431 |            break; |
| 6432 |          case ZEND_BIND_GLOBAL: |
| 6433 |            ssa_op++; |
| 6434 |            opline++; |
| 6435 |            while (opline->opcode == ZEND_BIND_GLOBAL) { |
| 6436 |              if (ssa_op->op1_def >= 0) { |
| 6437 |                zend_uchar type = IS_UNKNOWN; |
| 6438 | |
| 6439 |                if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD) |
| 6440 |                 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) { |
| 6441 |                  type = concrete_type(ssa->var_info[ssa_op->op1_def].type); |
| 6442 |                } |
| 6443 |                SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type, |
| 6444 |                  (!ra || !ra[ssa_op->op1_def])); |
| 6445 |                if (ra && ra[ssa_op->op1_def]) { |
| 6446 |                  SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def]->reg, |
| 6447 |                    ra[ssa_op->op1_def]->flags & ZREG_STORE); |
| 6448 |                } |
| 6449 |              } |
| 6450 |              ssa_op++; |
| 6451 |              opline++; |
| 6452 |            } |
| 6453 |            break; |
| 6454 |          default: |
| 6455 |            ssa_op += zend_jit_trace_op_len(opline); |
| 6456 |            break; |
| 6457 |        } |
| 6458 | |
| 6459 |        if (send_result) { |
| 6460 |          ssa_op++; |
| 6461 |          p++; |
| 6462 |          if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) { |
| 6463 |            p++; |
| 6464 |          } |
| 6465 |          send_result = 0; |
| 6466 |        } |
| 6467 |      } |
| 6468 |    } else if (p->op == ZEND_JIT_TRACE_ENTER) { |
| 6469 |      call = frame->call; |
| 6470 |      assert(call && &call->func->op_array == p->op_array); |
| 6471 | |
| 6472 |      if (opline->opcode == ZEND_DO_UCALL |
| 6473 |       || opline->opcode == ZEND_DO_FCALL_BY_NAME |
| 6474 |       || opline->opcode == ZEND_DO_FCALL) { |
| 6475 | |
| 6476 |        frame->call_opline = opline; |
| 6477 | |
| 6478 |        /* Check if SEND_UNPACK/SEND_ARRAY may cause enter at different opline */ |
| 6479 |        if (opline > op_array->opcodes) { |
| 6480 |          const zend_op *prev_opline = opline - 1; |
| 6481 | |
| 6482 |          while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { |
| 6483 |            prev_opline--; |
| 6484 |          } |
| 6485 |          JIT_G(current_frame) = call; |
| 6486 |          if ((prev_opline->opcode == ZEND_SEND_ARRAY |
| 6487 |           || prev_opline->opcode == ZEND_SEND_UNPACK |
| 6488 |           || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) |
| 6489 |           && p->op_array->num_args |
| 6490 |           && (p->op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0 |
| 6491 |           && ((p+1)->op == ZEND_JIT_TRACE_VM |
| 6492 |           || (p+1)->op == ZEND_JIT_TRACE_END) |
| 6493 |           && ( |
| 6494 |           ) |
| 6495 |           && !zend_jit_trace_opline_guard(&dasm_state, (p+1)->opline)) { |
| 6496 |            goto jit_failure; |
| 6497 |          } |
| 6498 |          JIT_G(current_frame) = frame; |
| 6499 |        } |
| 6500 |      } |
| 6501 | |
| 6502 |      if ((p+1)->op == ZEND_JIT_TRACE_END) { |
| 6503 |        p++; |
| 6504 |        break; |
| 6505 |      } |
| 6506 |      if (op_array->fn_flags & ZEND_ACC_CLOSURE) { |
| 6507 |        if (TRACE_FRAME_IS_THIS_CHECKED(frame)) { |
| 6508 |          TRACE_FRAME_SET_THIS_CHECKED(call); |
| 6509 |        } |
| 6510 |      } else if (op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC)) { |
| 6511 |        TRACE_FRAME_SET_THIS_CHECKED(call); |
| 6512 |      } |
| 6513 |      op_array = (zend_op_array*)p->op_array; |
| 6514 |      jit_extension = |
| 6515 |        (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 6516 |      op_array_ssa = &jit_extension->func_info.ssa; |
| 6517 |      frame->call = call->prev; |
| 6518 |      call->prev = frame; |
| 6519 |      if (p->info & ZEND_JIT_TRACE_RETURN_VALUE_USED) { |
| 6520 |        TRACE_FRAME_SET_RETURN_VALUE_USED(call); |
| 6521 |      } else { |
| 6522 |        TRACE_FRAME_SET_RETURN_VALUE_UNUSED(call); |
| 6523 |      } |
| 6524 |      JIT_G(current_frame) = frame = call; |
| 6525 |      stack = frame->stack; |
| 6526 |      if (ra) { |
| 6527 |        int j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); |
| 6528 | |
| 6529 |        for (i = 0; i < op_array->last_var; i++,j++) { |
| 6530 |          if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) { |
| 6531 |            SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD); |
| 6532 |            if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) { |
| 6533 |              goto jit_failure; |
| 6534 |            } |
| 6535 |          } |
| 6536 |        } |
| 6537 |      } |
| 6538 |    } else if (p->op == ZEND_JIT_TRACE_BACK) { |
| 6539 |      op_array = (zend_op_array*)p->op_array; |
| 6540 |      jit_extension = |
| 6541 |        (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 6542 |      op_array_ssa = &jit_extension->func_info.ssa; |
| 6543 |      top = frame; |
| 6544 |      if (frame->prev) { |
| 6545 |        checked_stack -= frame->used_stack; |
| 6546 |        frame = frame->prev; |
| 6547 |        stack = frame->stack; |
| 6548 |        ZEND_ASSERT(&frame->func->op_array == op_array); |
| 6549 |      } else { |
| 6550 |        frame = zend_jit_trace_ret_frame(frame, op_array); |
| 6551 |        TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1); |
| 6552 |        frame->used_stack = checked_stack = peek_checked_stack = 0; |
| 6553 |        stack = frame->stack; |
| 6554 |        if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { |
| 6555 |          uint32_t j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); |
| 6556 | |
| 6557 |          for (i = 0; i < op_array->last_var + op_array->T; i++, j++) { |
| 6558 |            /* Initialize abstract stack using SSA */ |
| 6559 |            if (!(ssa->var_info[j].type & MAY_BE_GUARD) |
| 6560 |             && has_concrete_type(ssa->var_info[j].type)) { |
| 6561 |              SET_STACK_TYPE(stack, i, concrete_type(ssa->var_info[j].type), 1); |
| 6562 |            } else { |
| 6563 |              SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1); |
| 6564 |            } |
| 6565 |          } |
| 6566 |          if (ra) { |
| 6567 |            j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); |
| 6568 |            for (i = 0; i < op_array->last_var + op_array->T; i++, j++) { |
| 6569 |              if (ra[j] && (ra[j]->flags & ZREG_LOAD) != 0) { |
| 6570 |                SET_STACK_REG_EX(stack, i, ra[j]->reg, ZREG_LOAD); |
| 6571 |                if (!zend_jit_load_var(&dasm_state, ssa->var_info[j].type, i, ra[j]->reg)) { |
| 6572 |                  goto jit_failure; |
| 6573 |                } |
| 6574 |              } |
| 6575 |            } |
| 6576 |          } |
| 6577 |        } else { |
| 6578 |          for (i = 0; i < op_array->last_var + op_array->T; i++) { |
| 6579 |            SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1); |
| 6580 |          } |
| 6581 |        } |
| 6582 |        opline = NULL; |
| 6583 |      } |
| 6584 |      JIT_G(current_frame) = frame; |
| 6585 |      if (res_type != IS_UNKNOWN |
| 6586 |       && (p+1)->op == ZEND_JIT_TRACE_VM) { |
| 6587 |        const zend_op *opline = (p+1)->opline - 1; |
| 6588 |        if (opline->result_type != IS_UNUSED) { |
| 6589 |         SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), res_type, 1); |
| 6590 |        } |
| 6591 |      } |
| 6592 |      res_type = IS_UNKNOWN; |
| 6593 |    } else if (p->op == ZEND_JIT_TRACE_END) { |
| 6594 |      break; |
| 6595 |    } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { |
| 6596 |      const zend_op *init_opline = zend_jit_trace_find_init_fcall_op(p, op_array); |
| 6597 |      int num_args = -1; |
| 6598 | |
| 6599 |      if (init_opline |
| 6600 |       && init_opline->extended_value <= TRACE_FRAME_MAX_NUM_ARGS) { |
| 6601 |        num_args = init_opline->extended_value; |
| 6602 |      } |
| 6603 | |
| 6604 |      call = top; |
| 6605 |      TRACE_FRAME_INIT(call, p->func, frame_flags, num_args); |
| 6606 |      call->prev = frame->call; |
| 6607 |      if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { |
| 6608 |        TRACE_FRAME_SET_LAST_SEND_BY_VAL(call); |
| 6609 |        if (init_opline && init_opline->opcode == ZEND_INIT_DYNAMIC_CALL) { |
| 6610 |          TRACE_FRAME_SET_CLOSURE_CALL(call); |
| 6611 |        } |
| 6612 |      } |
| 6613 |      if (init_opline) { |
| 6614 |        if (init_opline->opcode != ZEND_NEW |
| 6615 |         && (init_opline->opcode != ZEND_INIT_METHOD_CALL |
| 6616 |         || init_opline->op1_type == IS_UNDEF |
| 6617 |         || (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) |
| 6618 |         && (ssa_op-1)->op1_use >=0 |
| 6619 |         && .delayed_fetch_this)) |
| 6620 |         && (init_opline->opcode != ZEND_INIT_USER_CALL |
| 6621 |         || (p->func && ( ))) |
| 6622 |         && (init_opline->opcode != ZEND_INIT_DYNAMIC_CALL |
| 6623 |         || (p->func && ( ))) |
| 6624 |        ) { |
| 6625 |          TRACE_FRAME_SET_NO_NEED_RELEASE_THIS(call); |
| 6626 |        } else if (init_opline->opcode == ZEND_NEW |
| 6627 |         || (init_opline->opcode == ZEND_INIT_METHOD_CALL |
| 6628 |         && init_opline->op1_type != IS_UNDEF |
| 6629 |         && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) |
| 6630 |         && p->func && p->func->common.scope && !(p->func->common.fn_flags & ZEND_ACC_STATIC))) { |
| 6631 |          TRACE_FRAME_SET_ALWAYS_RELEASE_THIS(call); |
| 6632 |        } |
| 6633 |      } |
| 6634 |      frame->call = call; |
| 6635 |      top = zend_jit_trace_call_frame(top, p->op_array); |
| 6636 |      if (p->func) { |
| 6637 |        if (p->func->type == ZEND_USER_FUNCTION) { |
| 6638 |          if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { |
| 6639 |            zend_jit_op_array_trace_extension *jit_extension = |
| 6640 |              (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(p->op_array); |
| 6641 | |
| 6642 |            i = 0; |
| 6643 |            while (i < p->op_array->num_args) { |
| 6644 |              /* Types of arguments are going to be stored in abstract stack when processing SEV instruction */ |
| 6645 |              SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); |
| 6646 |              i++; |
| 6647 |            } |
| 6648 |            while (i < p->op_array->last_var) { |
| 6649 |              if (jit_extension |
| 6650 |               && zend_jit_var_may_alias(p->op_array, &jit_extension->func_info.ssa, i) != NO_ALIAS) { |
| 6651 |                SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); |
| 6652 |              } else { |
| 6653 |                SET_STACK_TYPE(call->stack, i, IS_UNDEF, 1); |
| 6654 |              } |
| 6655 |              i++; |
| 6656 |            } |
| 6657 |            while (i < p->op_array->last_var + p->op_array->T) { |
| 6658 |              SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); |
| 6659 |              i++; |
| 6660 |            } |
| 6661 |          } else { |
| 6662 |            for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) { |
| 6663 |              SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); |
| 6664 |            } |
| 6665 |          } |
| 6666 |        } else { |
| 6667 |          ZEND_ASSERT(p->func->type == ZEND_INTERNAL_FUNCTION); |
| 6668 |          for (i = 0; i < p->op_array->num_args; i++) { |
| 6669 |            SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1); |
| 6670 |          } |
| 6671 |        } |
| 6672 |        if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) { |
| 6673 |          int skip_guard = 0; |
| 6674 | |
| 6675 |          if (init_opline) { |
| 6676 |            zend_call_info *call_info = jit_extension->func_info.callee_info; |
| 6677 | |
| 6678 |            while (call_info) { |
| 6679 |              if (call_info->caller_init_opline == init_opline |
| 6680 |                  && !call_info->is_prototype) { |
| 6681 |                if (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) { |
| 6682 |                  if (init_opline->opcode == ZEND_INIT_STATIC_METHOD_CALL |
| 6683 |                   && init_opline->op1_type != IS_CONST) { |
| 6684 |                    break; |
| 6685 |                  } else if (init_opline->opcode == ZEND_INIT_METHOD_CALL) { |
| 6686 |                    break; |
| 6687 |                  } |
| 6688 |                } |
| 6689 |                skip_guard = 1; |
| 6690 |                break; |
| 6691 |              } |
| 6692 |              call_info = call_info->next_callee; |
| 6693 |            } |
| 6694 |            if (!skip_guard |
| 6695 |             && !zend_jit_may_be_polymorphic_call(init_opline) |
| 6696 |             && !zend_jit_may_be_modified(p->func, op_array)) { |
| 6697 |              skip_guard = 1; |
| 6698 |            } |
| 6699 |          } |
| 6700 | |
| 6701 |          if (!skip_guard) { |
| 6702 |            if (!opline) { |
| 6703 |              zend_jit_trace_rec *q = p + 1; |
| 6704 |              while (q->op != ZEND_JIT_TRACE_VM && q->op != ZEND_JIT_TRACE_END) { |
| 6705 |                q++; |
| 6706 |              } |
| 6707 |              opline = q->opline; |
| 6708 |              ZEND_ASSERT(opline != NULL); |
| 6709 |            } |
| 6710 |            if (!zend_jit_init_fcall_guard(&dasm_state, |
| 6711 |                ZEND_JIT_TRACE_FAKE_LEVEL(p->info), p->func, opline)) { |
| 6712 |              goto jit_failure; |
| 6713 |            } |
| 6714 |          } |
| 6715 |        } |
| 6716 |      } |
| 6717 |      if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) { |
| 6718 |        frame->call_level++; |
| 6719 |        call->used_stack = 0; |
| 6720 |      } else { |
| 6721 |        if (p->func) { |
| 6722 |          call->used_stack = zend_vm_calc_used_stack(init_opline->extended_value, (zend_function*)p->func); |
| 6723 |        } else { |
| 6724 |          call->used_stack = (ZEND_CALL_FRAME_SLOT + init_opline->extended_value) * sizeof(zval); |
| 6725 |        } |
| 6726 |        checked_stack += call->used_stack; |
| 6727 |        if (checked_stack > peek_checked_stack) { |
| 6728 |          peek_checked_stack = checked_stack; |
| 6729 |        } |
| 6730 |      } |
| 6731 |    } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { |
| 6732 |      call = frame->call; |
| 6733 |      if (call) { |
| 6734 |        checked_stack -= call->used_stack; |
| 6735 |        top = call; |
| 6736 |        frame->call = call->prev; |
| 6737 |      } |
| 6738 |    } else { |
| 6739 |      ZEND_UNREACHABLE(); |
| 6740 |    } |
| 6741 |  } |
| 6742 | |
| 6743 |  ZEND_ASSERT(p->op == ZEND_JIT_TRACE_END); |
| 6744 | |
| 6745 |  t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; |
| 6746 | |
| 6747 |  if (!parent_trace && zend_jit_trace_uses_initial_ip()) { |
| 6748 |    t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP; |
| 6749 |  } |
| 6750 | |
| 6751 |  if (p->stop == ZEND_JIT_TRACE_STOP_LOOP |
| 6752 |   || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL |
| 6753 |   || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 6754 |    if (ra) { |
| 6755 |      zend_ssa_phi *phi = ssa->blocks[1].phis; |
| 6756 | |
| 6757 |      while (phi) { |
| 6758 |        if (ra[phi->ssa_var] |
| 6759 |         && ra[phi->sources[1]] |
| 6760 |         && STACK_MEM_TYPE(stack, phi->var) != STACK_TYPE(stack, phi->var) |
| 6761 |         && (ra[phi->ssa_var]->flags & (ZREG_LOAD|ZREG_STORE)) == 0 |
| 6762 |         && (ra[phi->sources[1]]->flags & (ZREG_LOAD|ZREG_STORE)) == 0) { |
| 6763 |          /* Store actual type to memory to avoid deoptimization mistakes */ |
| 6764 |          /* TODO: Alternatively, we may try to update alredy generated deoptimization info */ |
| 6765 |          zend_jit_store_var_type(&dasm_state, phi->var, STACK_TYPE(stack, phi->var)); |
| 6766 |        } |
| 6767 |        phi = phi->next; |
| 6768 |      } |
| 6769 |    } |
| 6770 |    if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 6771 |      if ((t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP) |
| 6772 |       && !zend_jit_set_ip(&dasm_state, p->opline)) { |
| 6773 |        goto jit_failure; |
| 6774 |      } |
| 6775 |    } |
| 6776 |    t->link = ZEND_JIT_TRACE_NUM; |
| 6777 |    if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 6778 |      t->flags |= ZEND_JIT_TRACE_CHECK_INTERRUPT; |
| 6779 |    } |
| 6780 |    if (!(t->flags & ZEND_JIT_TRACE_LOOP)) { |
| 6781 |      const void *timeout_exit_addr = NULL; |
| 6782 | |
| 6783 |      t->flags |= ZEND_JIT_TRACE_LOOP; |
| 6784 | |
| 6785 |      if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 6786 |        if (!(t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP) |
| 6787 |         || (ra |
| 6788 |         && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T))) { |
| 6789 |          uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); |
| 6790 | |
| 6791 |          timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point); |
| 6792 |          if (!timeout_exit_addr) { |
| 6793 |            goto jit_failure; |
| 6794 |          } |
| 6795 |        } else { |
| 6796 |          timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler]; |
| 6797 |        } |
| 6798 |      } |
| 6799 | |
| 6800 |      zend_jit_trace_end_loop(&dasm_state, 0, timeout_exit_addr); /* jump back to start of the trace loop */ |
| 6801 |    } |
| 6802 |  } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK |
| 6803 |   || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { |
| 6804 |    if (!zend_jit_trace_deoptimization(&dasm_state, 0, NULL, |
| 6805 |        stack, op_array->last_var + op_array->T, NULL, NULL, NULL, 0)) { |
| 6806 |      goto jit_failure; |
| 6807 |    } |
| 6808 |    if (p->stop == ZEND_JIT_TRACE_STOP_LINK) { |
| 6809 |      const void *timeout_exit_addr = NULL; |
| 6810 | |
| 6811 |      t->link = zend_jit_find_trace(p->opline->handler); |
| 6812 |      if ((zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP) |
| 6813 |       && !zend_jit_set_ip(&dasm_state, p->opline)) { |
| 6814 |        goto jit_failure; |
| 6815 |      } |
| 6816 |      if (!parent_trace && zend_jit_trace_uses_initial_ip()) { |
| 6817 |        t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP; |
| 6818 |      } |
| 6819 |      if (parent_trace |
| 6820 |       && (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT) |
| 6821 |       && zend_jit_traces[parent_trace].root == t->link) { |
| 6822 |        if (!(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)) { |
| 6823 |          uint32_t exit_point; |
| 6824 | |
| 6825 |          for (i = 0; i < op_array->last_var + op_array->T; i++) { |
| 6826 |            SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1); |
| 6827 |          } |
| 6828 |          exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); |
| 6829 |          timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point); |
| 6830 |          if (!timeout_exit_addr) { |
| 6831 |            goto jit_failure; |
| 6832 |          } |
| 6833 |        } else { |
| 6834 |          timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler]; |
| 6835 |        } |
| 6836 |      } |
| 6837 |      zend_jit_trace_link_to_root(&dasm_state, &zend_jit_traces[t->link], timeout_exit_addr); |
| 6838 |    } else { |
| 6839 |      zend_jit_trace_return(&dasm_state, 0, NULL); |
| 6840 |    } |
| 6841 |  } else if (p->stop == ZEND_JIT_TRACE_STOP_RETURN) { |
| 6842 |    zend_jit_trace_return(&dasm_state, 0, NULL); |
| 6843 |  } else { |
| 6844 |    // TODO: not implemented ??? |
| 6845 |    ZEND_ASSERT(0 && p->stop); |
| 6846 |  } |
| 6847 | |
| 6848 |  if (ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) { |
| 6849 |    goto jit_failure; |
| 6850 |  } |
| 6851 | |
| 6852 |  if (!zend_jit_trace_end(&dasm_state, t)) { |
| 6853 |    goto jit_failure; |
| 6854 |  } |
| 6855 | |
| 6856 |  handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, ZSTR_VAL(name), ZEND_JIT_TRACE_NUM, |
| 6857 |    parent_trace ? SP_ADJ_JIT : ((zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) ? SP_ADJ_VM : SP_ADJ_RET), |
| 6858 |    parent_trace ? SP_ADJ_NONE : SP_ADJ_JIT); |
| 6859 | |
| 6860 |  if (handler) { |
| 6861 |    if (p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) { |
| 6862 |      const zend_op_array *rec_op_array; |
| 6863 | |
| 6864 |      rec_op_array = op_array = trace_buffer->op_array; |
| 6865 |      jit_extension = |
| 6866 |        (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 6867 |      p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; |
| 6868 |      for (;;p++) { |
| 6869 |        if (p->op == ZEND_JIT_TRACE_VM) { |
| 6870 |          opline = p->opline; |
| 6871 |        } else if (p->op == ZEND_JIT_TRACE_ENTER) { |
| 6872 |          if (p->op_array == rec_op_array) { |
| 6873 |            zend_jit_trace_setup_ret_counter(opline, jit_extension->offset); |
| 6874 |          } |
| 6875 |          op_array = p->op_array; |
| 6876 |          jit_extension = |
| 6877 |            (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 6878 |        } else if (p->op == ZEND_JIT_TRACE_BACK) { |
| 6879 |          op_array = p->op_array; |
| 6880 |          jit_extension = |
| 6881 |            (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 6882 |        } else if (p->op == ZEND_JIT_TRACE_END) { |
| 6883 |          break; |
| 6884 |        } |
| 6885 |      } |
| 6886 |    } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK |
| 6887 |     || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { |
| 6888 |      if (opline |
| 6889 |       && (opline->opcode == ZEND_DO_UCALL |
| 6890 |       || opline->opcode == ZEND_DO_FCALL |
| 6891 |       || opline->opcode == ZEND_DO_FCALL_BY_NAME |
| 6892 |       || opline->opcode == ZEND_YIELD |
| 6893 |       || opline->opcode == ZEND_YIELD_FROM |
| 6894 |       || opline->opcode == ZEND_INCLUDE_OR_EVAL)) { |
| 6895 |        zend_jit_trace_setup_ret_counter(opline, jit_extension->offset); |
| 6896 |      } |
| 6897 |      if (JIT_G(current_frame) |
| 6898 |       && JIT_G(current_frame)->prev) { |
| 6899 |        frame = JIT_G(current_frame)->prev; |
| 6900 |        do { |
| 6901 |          if (frame->call_opline) { |
| 6902 |            op_array = &frame->func->op_array; |
| 6903 |            jit_extension = |
| 6904 |              (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 6905 |            zend_jit_trace_setup_ret_counter(frame->call_opline, jit_extension->offset); |
| 6906 |          } |
| 6907 |          frame = frame->prev; |
| 6908 |        } while (frame); |
| 6909 |      } |
| 6910 |    } |
| 6911 |  } |
| 6912 | |
| 6913 | jit_failure: |
| 6914 |  dasm_free(&dasm_state); |
| 6915 | |
| 6916 |  if (name) { |
| 6917 |    zend_string_release(name); |
| 6918 |  } |
| 6919 | |
| 6920 | jit_cleanup: |
| 6921 |  /* Clean up used op_arrays */ |
| 6922 |  while (num_op_arrays > 0) { |
| 6923 |    op_array = op_arrays[--num_op_arrays]; |
| 6924 |    jit_extension = |
| 6925 |      (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 6926 | |
| 6927 |   jit_extension->func_info.num = 0; |
| 6928 |    jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC |
| 6929 |      | ZEND_FUNC_JIT_ON_PROF_REQUEST |
| 6930 |      | ZEND_FUNC_JIT_ON_HOT_COUNTERS |
| 6931 |      | ZEND_FUNC_JIT_ON_HOT_TRACE; |
| 6932 |    memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa)); |
| 6933 |  } |
| 6934 | |
| 6935 |  zend_arena_release(&CG(arena), checkpoint); |
| 6936 | |
| 6937 |  JIT_G(current_frame) = NULL; |
| 6938 |  JIT_G(current_trace) = NULL; |
| 6939 | |
| 6940 |  return handler; |
| 6941 | } |
| 6942 | |
| 6943 | static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num) |
| 6944 | { |
| 6945 |  const void *handler = NULL; |
| 6946 |  dasm_State* dasm_state = NULL; |
| 6947 |  void *checkpoint; |
| 6948 |  char name[32]; |
| 6949 |  const zend_op *opline; |
| 6950 |  uint32_t stack_size; |
| 6951 |  zend_jit_trace_stack *stack; |
| 6952 |  bool original_handler = 0; |
| 6953 | |
| 6954 |  if (!zend_jit_trace_exit_needs_deoptimization(trace_num, exit_num)) { |
| 6955 |    return dasm_labels[zend_lbtrace_escape]; |
| 6956 |  } |
| 6957 | |
| 6958 |  checkpoint = zend_arena_checkpoint(CG(arena));; |
| 6959 | |
| 6960 |  sprintf(name, "ESCAPE-%d-%d", trace_num, exit_num); |
| 6961 | |
| 6962 |  dasm_init(&dasm_state, DASM_MAXSECTION); |
| 6963 |  dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX); |
| 6964 |  dasm_setup(&dasm_state, dasm_actions); |
| 6965 | |
| 6966 |  zend_jit_align_func(&dasm_state); |
| 6967 | |
| 6968 |  /* Deoptimization */ |
| 6969 |  stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size; |
| 6970 |  stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset; |
| 6971 | |
| 6972 |  if (!zend_jit_trace_deoptimization(&dasm_state, |
| 6973 |      zend_jit_traces[trace_num].exit_info[exit_num].flags, |
| 6974 |      zend_jit_traces[trace_num].exit_info[exit_num].opline, |
| 6975 |      stack, stack_size, NULL, NULL, NULL, 0)) { |
| 6976 |    goto jit_failure; |
| 6977 |  } |
| 6978 | |
| 6979 |  opline = zend_jit_traces[trace_num].exit_info[exit_num].opline; |
| 6980 |  if (opline) { |
| 6981 |    if (opline == zend_jit_traces[zend_jit_traces[trace_num].root].opline) { |
| 6982 |      /* prevent endless loop */ |
| 6983 |      original_handler = 1; |
| 6984 |    } |
| 6985 |    zend_jit_set_ip_ex(&dasm_state, opline, original_handler); |
| 6986 |  } |
| 6987 | |
| 6988 |  zend_jit_trace_return(&dasm_state, original_handler, opline); |
| 6989 | |
| 6990 |  handler = dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, name, ZEND_JIT_TRACE_NUM, SP_ADJ_JIT, SP_ADJ_NONE); |
| 6991 | |
| 6992 | jit_failure: |
| 6993 |  dasm_free(&dasm_state); |
| 6994 |  zend_arena_release(&CG(arena), checkpoint); |
| 6995 |  return handler; |
| 6996 | } |
| 6997 | |
| 6998 | static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset) |
| 6999 | { |
| 7000 |  zend_jit_trace_stop ret; |
| 7001 |  const void *handler; |
| 7002 |  uint8_t orig_trigger; |
| 7003 |  zend_jit_trace_info *t = NULL; |
| 7004 |  zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS]; |
| 7005 | |
| 7006 |  zend_shared_alloc_lock(); |
| 7007 | |
| 7008 |  /* Checks under lock */ |
| 7009 |  if ((ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_JITED)) { |
| 7010 |    ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE; |
| 7011 |  } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) { |
| 7012 |    ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES; |
| 7013 |  } else { |
| 7014 |    SHM_UNPROTECT(); |
| 7015 |    zend_jit_unprotect(); |
| 7016 | |
| 7017 |    t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; |
| 7018 | |
| 7019 |    t->id = ZEND_JIT_TRACE_NUM; |
| 7020 |    t->root = ZEND_JIT_TRACE_NUM; |
| 7021 |    t->parent = 0; |
| 7022 |    t->link = 0; |
| 7023 |    t->exit_count = 0; |
| 7024 |    t->child_count = 0; |
| 7025 |    t->stack_map_size = 0; |
| 7026 |    t->flags = 0; |
| 7027 |    t->polymorphism = 0; |
| 7028 |    t->jmp_table_size = 0; |
| 7029 |    t->op_array = trace_buffer[0].op_array; |
| 7030 |    t->opline = trace_buffer[1].opline; |
| 7031 |    t->exit_info = exit_info; |
| 7032 |    t->stack_map = NULL; |
| 7033 | |
| 7034 |    orig_trigger = JIT_G(trigger); |
| 7035 |    JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE; |
| 7036 | |
| 7037 |    handler = zend_jit_trace(trace_buffer, 0, 0); |
| 7038 | |
| 7039 |    JIT_G(trigger) = orig_trigger; |
| 7040 | |
| 7041 |    if (handler) { |
| 7042 |      zend_jit_trace_exit_info *shared_exit_info = NULL; |
| 7043 | |
| 7044 |      t->exit_info = NULL; |
| 7045 |      if (t->exit_count) { |
| 7046 |        /* reallocate exit_info into shared memory */ |
| 7047 |        shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc( |
| 7048 |          sizeof(zend_jit_trace_exit_info) * t->exit_count); |
| 7049 | |
| 7050 |        if (!shared_exit_info) { |
| 7051 |         if (t->stack_map) { |
| 7052 |            efree(t->stack_map); |
| 7053 |            t->stack_map = NULL; |
| 7054 |          } |
| 7055 |          ret = ZEND_JIT_TRACE_STOP_NO_SHM; |
| 7056 |          goto exit; |
| 7057 |        } |
| 7058 |        memcpy(shared_exit_info, exit_info, |
| 7059 |          sizeof(zend_jit_trace_exit_info) * t->exit_count); |
| 7060 |        t->exit_info = shared_exit_info; |
| 7061 |      } |
| 7062 | |
| 7063 |     if (t->stack_map_size) { |
| 7064 |        zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack)); |
| 7065 |        if (!shared_stack_map) { |
| 7066 |          ret = ZEND_JIT_TRACE_STOP_NO_SHM; |
| 7067 |          goto exit; |
| 7068 |        } |
| 7069 |        memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack)); |
| 7070 |        efree(t->stack_map); |
| 7071 |        t->stack_map = shared_stack_map; |
| 7072 |     } |
| 7073 | |
| 7074 |      t->exit_counters = ZEND_JIT_EXIT_COUNTERS; |
| 7075 |      ZEND_JIT_EXIT_COUNTERS += t->exit_count; |
| 7076 | |
| 7077 |      ((zend_op*)opline)->handler = handler; |
| 7078 | |
| 7079 |      ZEND_JIT_TRACE_NUM++; |
| 7080 |      ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_JITED; |
| 7081 | |
| 7082 |      ret = ZEND_JIT_TRACE_STOP_COMPILED; |
| 7083 |    } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS || |
| 7084 |     ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) { |
| 7085 |     if (t->stack_map) { |
| 7086 |        efree(t->stack_map); |
| 7087 |        t->stack_map = NULL; |
| 7088 |      } |
| 7089 |      ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS; |
| 7090 |    } else { |
| 7091 |     if (t->stack_map) { |
| 7092 |        efree(t->stack_map); |
| 7093 |        t->stack_map = NULL; |
| 7094 |      } |
| 7095 |      ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR; |
| 7096 |    } |
| 7097 | |
| 7098 | exit: |
| 7099 |    zend_jit_protect(); |
| 7100 |    SHM_PROTECT(); |
| 7101 |  } |
| 7102 | |
| 7103 |  zend_shared_alloc_unlock(); |
| 7104 | |
| 7105 |  if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0 |
| 7106 |   && ret == ZEND_JIT_TRACE_STOP_COMPILED |
| 7107 |   && t->exit_count > 0) { |
| 7108 |    zend_jit_dump_exit_info(t); |
| 7109 |  } |
| 7110 | |
| 7111 |  return ret; |
| 7112 | } |
| 7113 | |
| 7114 | static void zend_jit_blacklist_root_trace(const zend_op *opline, size_t offset) |
| 7115 | { |
| 7116 |  zend_shared_alloc_lock(); |
| 7117 | |
| 7118 |  if (!(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) { |
| 7119 |    SHM_UNPROTECT(); |
| 7120 |    zend_jit_unprotect(); |
| 7121 | |
| 7122 |    ((zend_op*)opline)->handler = |
| 7123 |      ZEND_OP_TRACE_INFO(opline, offset)->orig_handler; |
| 7124 | |
| 7125 |    ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED; |
| 7126 | |
| 7127 |    zend_jit_protect(); |
| 7128 |    SHM_PROTECT(); |
| 7129 |  } |
| 7130 | |
| 7131 |  zend_shared_alloc_unlock(); |
| 7132 | } |
| 7133 | |
| 7134 | static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_stop stop, size_t offset) |
| 7135 | { |
| 7136 |  const zend_op **cache_opline = JIT_G(bad_root_cache_opline); |
| 7137 |  uint8_t *cache_count = JIT_G(bad_root_cache_count); |
| 7138 |  uint8_t *cache_stop = JIT_G(bad_root_cache_stop); |
| 7139 |  uint32_t cache_slot = JIT_G(bad_root_slot); |
| 7140 |  uint32_t i; |
| 7141 | |
| 7142 |  for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) { |
| 7143 |    if (cache_opline[i] == opline) { |
| 7144 |      if (cache_count[i] >= JIT_G(blacklist_root_trace) - 1) { |
| 7145 |        cache_opline[i] = NULL; |
| 7146 |        return 1; |
| 7147 |      } else { |
| 7148 | #if 0 |
| 7149 |        if (ZEND_OP_TRACE_INFO(opline, offset)->counter) { |
| 7150 |          *ZEND_OP_TRACE_INFO(opline, offset)->counter = |
| 7151 |            random() % ZEND_JIT_TRACE_COUNTER_MAX; |
| 7152 |        } |
| 7153 | #endif |
| 7154 |        cache_count[i]++; |
| 7155 |        cache_stop[i] = stop; |
| 7156 |        return 0; |
| 7157 |      } |
| 7158 |    } |
| 7159 |  } |
| 7160 |  i = cache_slot; |
| 7161 |  cache_opline[i] = opline; |
| 7162 |  cache_count[i] = 1; |
| 7163 |  cache_stop[i] = stop; |
| 7164 |  cache_slot = (i + 1) % ZEND_JIT_TRACE_BAD_ROOT_SLOTS; |
| 7165 |  JIT_G(bad_root_slot) = cache_slot; |
| 7166 |  return 0; |
| 7167 | } |
| 7168 | |
| 7169 | static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa) |
| 7170 | { |
| 7171 |  zend_jit_trace_rec *p = trace_buffer; |
| 7172 |  const zend_op_array *op_array; |
| 7173 |  const zend_op *opline; |
| 7174 |  uint32_t level = 1 + trace_buffer[0].level; |
| 7175 |  int idx, len, i, v, vars_count, call_level; |
| 7176 | |
| 7177 |  ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START); |
| 7178 |  op_array = p->op_array; |
| 7179 |  p += ZEND_JIT_TRACE_START_REC_SIZE; |
| 7180 |  idx = 0; |
| 7181 |  call_level = 0; |
| 7182 | |
| 7183 |  if (tssa && tssa->var_info) { |
| 7184 |    if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) { |
| 7185 |      vars_count = op_array->last_var; |
| 7186 |    } else { |
| 7187 |      vars_count = op_array->last_var + op_array->T; |
| 7188 |    } |
| 7189 |    for (i = 0; i < vars_count; i++) { |
| 7190 |      if (tssa->vars[i].use_chain >= 0 || tssa->vars[i].phi_use_chain) { |
| 7191 |        fprintf(stderr, " %*c;", level, ' '); |
| 7192 |        zend_dump_ssa_var(op_array, tssa, i, 0, i, ZEND_DUMP_RC_INFERENCE); |
| 7193 |        fprintf(stderr, "\n"); |
| 7194 |      } |
| 7195 |    } |
| 7196 |    if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP |
| 7197 |     || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL |
| 7198 |     || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { |
| 7199 |      zend_ssa_phi *p = tssa->blocks[1].phis; |
| 7200 | |
| 7201 |      fprintf(stderr, "LOOP:\n"); |
| 7202 | |
| 7203 |      while (p) { |
| 7204 |        fprintf(stderr, " ;"); |
| 7205 |        zend_dump_ssa_var(op_array, tssa, p->ssa_var, 0, p->var, ZEND_DUMP_RC_INFERENCE); |
| 7206 |        fprintf(stderr, " = Phi("); |
| 7207 |        zend_dump_ssa_var(op_array, tssa, p->sources[0], 0, p->var, ZEND_DUMP_RC_INFERENCE); |
| 7208 |        fprintf(stderr, ", "); |
| 7209 |        zend_dump_ssa_var(op_array, tssa, p->sources[1], 0, p->var, ZEND_DUMP_RC_INFERENCE); |
| 7210 |        fprintf(stderr, ")\n"); |
| 7211 |        p = p->next; |
| 7212 |      } |
| 7213 |    } |
| 7214 |  } |
| 7215 | |
| 7216 |  while (1) { |
| 7217 |    if (p->op == ZEND_JIT_TRACE_VM) { |
| 7218 |      uint8_t op1_type, op2_type, op3_type; |
| 7219 | |
| 7220 |      opline = p->opline; |
| 7221 |      fprintf(stderr, "%04d%*c", |
| 7222 |        (int)(opline - op_array->opcodes), |
| 7223 |        level, ' '); |
| 7224 |      zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL); |
| 7225 | |
| 7226 |      op1_type = p->op1_type; |
| 7227 |      op2_type = p->op2_type; |
| 7228 |      op3_type = p->op3_type; |
| 7229 |      if (op1_type != IS_UNKNOWN || op2_type != IS_UNKNOWN || op3_type != IS_UNKNOWN) { |
| 7230 |        fprintf(stderr, " ;"); |
| 7231 |        if (op1_type != IS_UNKNOWN) { |
| 7232 |          const char *ref = (op1_type & IS_TRACE_INDIRECT) ? |
| 7233 |            ((op1_type & IS_TRACE_REFERENCE) ? "*&" : "*") : |
| 7234 |            ((op1_type & IS_TRACE_REFERENCE) ? "&" : ""); |
| 7235 |          if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) { |
| 7236 |            p++; |
| 7237 |            fprintf(stderr, " op1(%sobject of class %s)", ref, |
| 7238 |              ZSTR_VAL(p->ce->name)); |
| 7239 |          } else { |
| 7240 |            const char *type = (op1_type == 0) ? "undef" : zend_get_type_by_const(op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)); |
| 7241 |            fprintf(stderr, " op1(%s%s%s)", ref, (op1_type & IS_TRACE_PACKED) ? "packed " : "", type); |
| 7242 |          } |
| 7243 |        } |
| 7244 |        if (op2_type != IS_UNKNOWN) { |
| 7245 |          const char *ref = (op2_type & IS_TRACE_INDIRECT) ? |
| 7246 |            ((op2_type & IS_TRACE_REFERENCE) ? "*&" : "*") : |
| 7247 |            ((op2_type & IS_TRACE_REFERENCE) ? "&" : ""); |
| 7248 |          if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) { |
| 7249 |            p++; |
| 7250 |            fprintf(stderr, " op2(%sobject of class %s)", ref, |
| 7251 |              ZSTR_VAL(p->ce->name)); |
| 7252 |          } else { |
| 7253 |            const char *type = (op2_type == 0) ? "undef" : zend_get_type_by_const(op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)); |
| 7254 |            fprintf(stderr, " op2(%s%s)", ref, type); |
| 7255 |          } |
| 7256 |        } |
| 7257 |        if (op3_type != IS_UNKNOWN) { |
| 7258 |          const char *ref = (op3_type & IS_TRACE_INDIRECT) ? |
| 7259 |            ((op3_type & IS_TRACE_REFERENCE) ? "*&" : "*") : |
| 7260 |            ((op3_type & IS_TRACE_REFERENCE) ? "&" : ""); |
| 7261 |          const char *type = (op3_type == 0) ? "undef" : zend_get_type_by_const(op3_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)); |
| 7262 |          fprintf(stderr, " op3(%s%s)", ref, type); |
| 7263 |        } |
| 7264 |      } |
| 7265 |      if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) { |
| 7266 |        uint8_t val_type; |
| 7267 |        const char *type; |
| 7268 | |
| 7269 |        if (op1_type == IS_UNKNOWN && op2_type == IS_UNKNOWN && op3_type == IS_UNKNOWN) { |
| 7270 |          fprintf(stderr, " ;"); |
| 7271 |        } |
| 7272 |        p++; |
| 7273 |        val_type = p->op1_type; |
| 7274 | |
| 7275 |        if (val_type == IS_UNDEF) { |
| 7276 |          type = "undef"; |
| 7277 |        } else if (val_type == IS_REFERENCE) { |
| 7278 |          type = "ref"; |
| 7279 |        } else { |
| 7280 |          type = zend_get_type_by_const(val_type); |
| 7281 |        } |
| 7282 |        fprintf(stderr, " val(%s)", type); |
| 7283 |      } |
| 7284 |      fprintf(stderr, "\n"); |
| 7285 |      idx++; |
| 7286 | |
| 7287 |      len = zend_jit_trace_op_len(opline); |
| 7288 |      while (len > 1) { |
| 7289 |        opline++; |
| 7290 |        fprintf(stderr, "%04d%*c;", |
| 7291 |          (int)(opline - op_array->opcodes), |
| 7292 |          level, ' '); |
| 7293 |        zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL); |
| 7294 |        idx++; |
| 7295 |        len--; |
| 7296 |        fprintf(stderr, "\n"); |
| 7297 |      } |
| 7298 |    } else if (p->op == ZEND_JIT_TRACE_ENTER) { |
| 7299 |      op_array = p->op_array; |
| 7300 |      fprintf(stderr, " %*c>enter %s%s%s\n", |
| 7301 |        level, ' ', |
| 7302 |        op_array->scope ? ZSTR_VAL(op_array->scope->name) : "", |
| 7303 |        op_array->scope ? "::" : "", |
| 7304 |        op_array->function_name ? |
| 7305 |          ZSTR_VAL(op_array->function_name) : |
| 7306 |          ZSTR_VAL(op_array->filename)); |
| 7307 |      level++; |
| 7308 |      if (tssa && tssa->var_info) { |
| 7309 |        call_level++; |
| 7310 |        v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); |
| 7311 |        vars_count = op_array->last_var; |
| 7312 |        for (i = 0; i < vars_count; i++, v++) { |
| 7313 |          if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) { |
| 7314 |            fprintf(stderr, " %*c;", level, ' '); |
| 7315 |            zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE); |
| 7316 |            fprintf(stderr, "\n"); |
| 7317 |          } |
| 7318 |        } |
| 7319 |      } |
| 7320 |    } else if (p->op == ZEND_JIT_TRACE_BACK) { |
| 7321 |      op_array = p->op_array; |
| 7322 |      level--; |
| 7323 |      fprintf(stderr, " %*c<back %s%s%s\n", |
| 7324 |        level, ' ', |
| 7325 |        op_array->scope ? ZSTR_VAL(op_array->scope->name) : "", |
| 7326 |        op_array->scope ? "::" : "", |
| 7327 |        op_array->function_name ? |
| 7328 |          ZSTR_VAL(op_array->function_name) : |
| 7329 |          ZSTR_VAL(op_array->filename)); |
| 7330 |      if (tssa && tssa->var_info) { |
| 7331 |        if (call_level == 0) { |
| 7332 |          v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info); |
| 7333 |          vars_count = op_array->last_var + op_array->T; |
| 7334 |          for (i = 0; i < vars_count; i++, v++) { |
| 7335 |            if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) { |
| 7336 |              fprintf(stderr, " %*c;", level, ' '); |
| 7337 |              zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE); |
| 7338 |              fprintf(stderr, "\n"); |
| 7339 |            } |
| 7340 |          } |
| 7341 |        } else { |
| 7342 |          call_level--; |
| 7343 |        } |
| 7344 |      } |
| 7345 |    } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { |
| 7346 |      if (p->func != (zend_function*)&zend_pass_function) { |
| 7347 |        fprintf(stderr, (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) ? " %*c>fake_init %s%s%s\n" : " %*c>init %s%s%s\n", |
| 7348 |          level, ' ', |
| 7349 |          (p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "", |
| 7350 |          (p->func && p->func->common.scope) ? "::" : "", |
| 7351 |          p->func ? ZSTR_VAL(p->func->common.function_name) : "???"); |
| 7352 |      } else { |
| 7353 |        fprintf(stderr, " %*c>skip\n", |
| 7354 |          level, ' '); |
| 7355 |      } |
| 7356 |    } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { |
| 7357 |      if (p->func != (zend_function*)&zend_pass_function) { |
| 7358 |        fprintf(stderr, " %*c>call %s%s%s\n", |
| 7359 |          level, ' ', |
| 7360 |          p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "", |
| 7361 |          p->func->common.scope ? "::" : "", |
| 7362 |          ZSTR_VAL(p->func->common.function_name)); |
| 7363 |      } else { |
| 7364 |        fprintf(stderr, " %*c>skip\n", |
| 7365 |          level, ' '); |
| 7366 |      } |
| 7367 |    } else if (p->op == ZEND_JIT_TRACE_END) { |
| 7368 |      break; |
| 7369 |    } |
| 7370 |    p++; |
| 7371 |  } |
| 7372 | } |
| 7373 | |
| 7374 | static void zend_jit_dump_exit_info(zend_jit_trace_info *t) |
| 7375 | { |
| 7376 |  int i, j; |
| 7377 | |
| 7378 |  fprintf(stderr, "---- TRACE %d exit info\n", t->id); |
| 7379 |  for (i = 0; i < t->exit_count; i++) { |
| 7380 |    const zend_op_array *op_array = t->exit_info[i].op_array; |
| 7381 |    uint32_t stack_size = t->exit_info[i].stack_size; |
| 7382 |    zend_jit_trace_stack *stack = t->stack_map + t->exit_info[i].stack_offset; |
| 7383 | |
| 7384 |    fprintf(stderr, " exit_%d:", i); |
| 7385 |    if (t->exit_info[i].opline) { |
| 7386 |      fprintf(stderr, " %04d/", (int)(t->exit_info[i].opline - op_array->opcodes)); |
| 7387 |    } else { |
| 7388 |      fprintf(stderr, " ----/"); |
| 7389 |    } |
| 7390 |    if (t->exit_info[i].stack_size) { |
| 7391 |      fprintf(stderr, "%04d/%d", t->exit_info[i].stack_offset, t->exit_info[i].stack_size); |
| 7392 |    } else { |
| 7393 |      fprintf(stderr, "----/0"); |
| 7394 |    } |
| 7395 |    if (t->exit_info[i].flags & ZEND_JIT_EXIT_TO_VM) { |
| 7396 |      fprintf(stderr, "/VM"); |
| 7397 |    } |
| 7398 |    if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) { |
| 7399 |      fprintf(stderr, "/CALL"); |
| 7400 |    } |
| 7401 |    if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) { |
| 7402 |      fprintf(stderr, "/POLY"); |
| 7403 |    } |
| 7404 |    if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP1) { |
| 7405 |      fprintf(stderr, "/FREE_OP1"); |
| 7406 |    } |
| 7407 |    if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP2) { |
| 7408 |      fprintf(stderr, "/FREE_OP2"); |
| 7409 |    } |
| 7410 |    for (j = 0; j < stack_size; j++) { |
| 7411 |      zend_uchar type = STACK_TYPE(stack, j); |
| 7412 |      if (type != IS_UNKNOWN) { |
| 7413 |        fprintf(stderr, " "); |
| 7414 |        zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j); |
| 7415 |        fprintf(stderr, ":"); |
| 7416 |        if (type == IS_UNDEF) { |
| 7417 |          fprintf(stderr, "undef"); |
| 7418 |        } else { |
| 7419 |          fprintf(stderr, "%s", zend_get_type_by_const(type)); |
| 7420 |        } |
| 7421 |        if (STACK_REG(stack, j) != ZREG_NONE) { |
| 7422 |          if (STACK_REG(stack, j) < ZREG_NUM) { |
| 7423 |            fprintf(stderr, "(%s)", zend_reg_name[STACK_REG(stack, j)]); |
| 7424 |          } else if (STACK_REG(stack, j) == ZREG_THIS) { |
| 7425 |            fprintf(stderr, "(this)"); |
| 7426 |          } else if (STACK_REG(stack, j) == ZREG_ZVAL_TRY_ADDREF) { |
| 7427 |            fprintf(stderr, "(zval_try_addref)"); |
| 7428 |          } else { |
| 7429 |            fprintf(stderr, "(const_%d)", STACK_REG(stack, j) - ZREG_NUM); |
| 7430 |          } |
| 7431 |        } |
| 7432 |      } else if (STACK_REG(stack, j) == ZREG_ZVAL_TRY_ADDREF) { |
| 7433 |        fprintf(stderr, " "); |
| 7434 |        zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j); |
| 7435 |        fprintf(stderr, ":unknown(zval_try_addref)"); |
| 7436 |      } else if (STACK_REG(stack, j) == ZREG_ZVAL_COPY_GPR0) { |
| 7437 |        fprintf(stderr, " "); |
| 7438 |        zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j); |
| 7439 |        fprintf(stderr, ":unknown(zval_copy(%s))", zend_reg_name[ZREG_COPY]); |
| 7440 |      } |
| 7441 |    } |
| 7442 |    fprintf(stderr, "\n"); |
| 7443 |  } |
| 7444 | } |
| 7445 | |
| 7446 | int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline) |
| 7447 | { |
| 7448 |  const zend_op *orig_opline; |
| 7449 |  zend_jit_trace_stop stop; |
| 7450 |  int ret = 0; |
| 7451 |  zend_op_array *op_array; |
| 7452 |  zend_jit_op_array_trace_extension *jit_extension; |
| 7453 |  size_t offset; |
| 7454 |  uint32_t trace_num; |
| 7455 |  zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH]; |
| 7456 | |
| 7457 |  ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION); |
| 7458 |  ZEND_ASSERT(opline >= EX(func)->op_array.opcodes && |
| 7459 |    opline < EX(func)->op_array.opcodes + EX(func)->op_array.last); |
| 7460 | |
| 7461 | repeat: |
| 7462 |  trace_num = ZEND_JIT_TRACE_NUM; |
| 7463 |  orig_opline = opline; |
| 7464 |  op_array = &EX(func)->op_array; |
| 7465 |  jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 7466 |  offset = jit_extension->offset; |
| 7467 | |
| 7468 |  EX(opline) = opline; |
| 7469 | |
| 7470 |  /* Lock-free check if the root trace was already JIT-ed or blacklist-ed in another process */ |
| 7471 |  if (ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) { |
| 7472 |    return 0; |
| 7473 |  } |
| 7474 | |
| 7475 |  if (JIT_G(tracing)) { |
| 7476 |    ++(*ZEND_OP_TRACE_INFO(opline, offset)->counter); |
| 7477 |    return 0; |
| 7478 |  } |
| 7479 | |
| 7480 |  if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) { |
| 7481 |    fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n", |
| 7482 |      trace_num, |
| 7483 |      zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags), |
| 7484 |      EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "", |
| 7485 |      EX(func)->op_array.scope ? "::" : "", |
| 7486 |      EX(func)->op_array.function_name ? |
| 7487 |        ZSTR_VAL(EX(func)->op_array.function_name) : "$main", |
| 7488 |      ZSTR_VAL(EX(func)->op_array.filename), |
| 7489 |      opline->lineno); |
| 7490 |  } |
| 7491 | |
| 7492 |  if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) { |
| 7493 |    stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES; |
| 7494 |    goto abort; |
| 7495 |  } |
| 7496 | |
| 7497 |  JIT_G(tracing) = 1; |
| 7498 |  stop = zend_jit_trace_execute(execute_data, opline, trace_buffer, |
| 7499 |    ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0); |
| 7500 |  JIT_G(tracing) = 0; |
| 7501 | |
| 7502 |  if (stop & ZEND_JIT_TRACE_HALT) { |
| 7503 |    ret = -1; |
| 7504 |  } |
| 7505 |  stop &= ~ZEND_JIT_TRACE_HALT; |
| 7506 | |
| 7507 |  if (UNEXPECTED(trace_buffer[1].opline != orig_opline)) { |
| 7508 |    orig_opline = trace_buffer[1].opline; |
| 7509 |    op_array = (zend_op_array*)trace_buffer[0].op_array; |
| 7510 |    jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 7511 |    offset = jit_extension->offset; |
| 7512 |    if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) { |
| 7513 |      const zend_op_array *op_array = trace_buffer[0].op_array; |
| 7514 |      const zend_op *opline = trace_buffer[1].opline; |
| 7515 |      zend_jit_op_array_trace_extension *jit_extension = |
| 7516 |        (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 7517 |      size_t offset = jit_extension->offset; |
| 7518 | |
| 7519 |      fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n", |
| 7520 |        trace_num, |
| 7521 |        zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags), |
| 7522 |        op_array->scope ? ZSTR_VAL(op_array->scope->name) : "", |
| 7523 |        op_array->scope ? "::" : "", |
| 7524 |        op_array->function_name ? |
| 7525 |          ZSTR_VAL(op_array->function_name) : "$main", |
| 7526 |        ZSTR_VAL(op_array->filename), |
| 7527 |        opline->lineno); |
| 7528 |    } |
| 7529 |  } |
| 7530 | |
| 7531 |  if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) { |
| 7532 |    zend_jit_dump_trace(trace_buffer, NULL); |
| 7533 |  } |
| 7534 | |
| 7535 |  if (ZEND_JIT_TRACE_STOP_OK(stop)) { |
| 7536 |    if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) { |
| 7537 |      if (stop == ZEND_JIT_TRACE_STOP_LINK) { |
| 7538 |        uint32_t idx = trace_buffer[1].last; |
| 7539 |        uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler); |
| 7540 |        fprintf(stderr, "---- TRACE %d stop (link to %d)\n", |
| 7541 |          trace_num, |
| 7542 |          link_to); |
| 7543 |      } else { |
| 7544 |        fprintf(stderr, "---- TRACE %d stop (%s)\n", |
| 7545 |          trace_num, |
| 7546 |          zend_jit_trace_stop_description[stop]); |
| 7547 |      } |
| 7548 |    } |
| 7549 |    stop = zend_jit_compile_root_trace(trace_buffer, orig_opline, offset); |
| 7550 |    if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) { |
| 7551 |      if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) { |
| 7552 |        fprintf(stderr, "---- TRACE %d %s\n", |
| 7553 |          trace_num, |
| 7554 |          zend_jit_trace_stop_description[stop]); |
| 7555 |      } |
| 7556 |    } else { |
| 7557 |      goto abort; |
| 7558 |    } |
| 7559 |  } else { |
| 7560 | abort: |
| 7561 |    if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) { |
| 7562 |      fprintf(stderr, "---- TRACE %d abort (%s)\n", |
| 7563 |        trace_num, |
| 7564 |        zend_jit_trace_stop_description[stop]); |
| 7565 |    } |
| 7566 |    if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop) |
| 7567 |     || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) { |
| 7568 |      if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) { |
| 7569 |        fprintf(stderr, "---- TRACE %d blacklisted\n", |
| 7570 |          trace_num); |
| 7571 |      } |
| 7572 |      zend_jit_blacklist_root_trace(orig_opline, offset); |
| 7573 |    } |
| 7574 |    if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) { |
| 7575 |      execute_data = EG(current_execute_data); |
| 7576 |      opline = EX(opline); |
| 7577 |      goto repeat; |
| 7578 |    } |
| 7579 |  } |
| 7580 | |
| 7581 |  if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) { |
| 7582 |    fprintf(stderr, "\n"); |
| 7583 |  } |
| 7584 | |
| 7585 |  return ret; |
| 7586 | } |
| 7587 | |
| 7588 | static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num) |
| 7589 | { |
| 7590 |  const void *handler; |
| 7591 | |
| 7592 |  zend_shared_alloc_lock(); |
| 7593 | |
| 7594 |  if (!(zend_jit_traces[trace_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED))) { |
| 7595 |    SHM_UNPROTECT(); |
| 7596 |    zend_jit_unprotect(); |
| 7597 | |
| 7598 |    handler = zend_jit_trace_exit_to_vm(trace_num, exit_num); |
| 7599 | |
| 7600 |    if (handler) { |
| 7601 |      zend_jit_link_side_trace( |
| 7602 |        zend_jit_traces[trace_num].code_start, |
| 7603 |        zend_jit_traces[trace_num].code_size, |
| 7604 |        zend_jit_traces[trace_num].jmp_table_size, |
| 7605 |        exit_num, |
| 7606 |        handler); |
| 7607 |    } |
| 7608 | |
| 7609 |    zend_jit_traces[trace_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_BLACKLISTED; |
| 7610 | |
| 7611 |    zend_jit_protect(); |
| 7612 |    SHM_PROTECT(); |
| 7613 |  } |
| 7614 | |
| 7615 |  zend_shared_alloc_unlock(); |
| 7616 | } |
| 7617 | |
| 7618 | static bool zend_jit_trace_exit_is_bad(uint32_t trace_num, uint32_t exit_num) |
| 7619 | { |
| 7620 |  uint8_t *counter = JIT_G(exit_counters) + |
| 7621 |    zend_jit_traces[trace_num].exit_counters + exit_num; |
| 7622 | |
| 7623 |  if ( ) { |
| 7624 |    return 1; |
| 7625 |  } |
| 7626 |  (*counter)++; |
| 7627 |  return 0; |
| 7628 | } |
| 7629 | |
| 7630 | static bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num) |
| 7631 | { |
| 7632 |  uint8_t *counter = JIT_G(exit_counters) + |
| 7633 |    zend_jit_traces[trace_num].exit_counters + exit_num; |
| 7634 | |
| 7635 |  if ( ) { |
| 7636 |    return 1; |
| 7637 |  } |
| 7638 |  (*counter)++; |
| 7639 |  return 0; |
| 7640 | } |
| 7641 | |
| 7642 | static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism) |
| 7643 | { |
| 7644 |  zend_jit_trace_stop ret; |
| 7645 |  const void *handler; |
| 7646 |  uint8_t orig_trigger; |
| 7647 |  zend_jit_trace_info *t; |
| 7648 |  zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS]; |
| 7649 | |
| 7650 |  zend_shared_alloc_lock(); |
| 7651 | |
| 7652 |  /* Checks under lock */ |
| 7653 |  if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) { |
| 7654 |    ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE; |
| 7655 |  } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) { |
| 7656 |    ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES; |
| 7657 |  } else if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) { |
| 7658 |    ret = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN; |
| 7659 |  } else { |
| 7660 |    SHM_UNPROTECT(); |
| 7661 |    zend_jit_unprotect(); |
| 7662 | |
| 7663 |    t = &zend_jit_traces[ZEND_JIT_TRACE_NUM]; |
| 7664 | |
| 7665 |    t->id = ZEND_JIT_TRACE_NUM; |
| 7666 |    t->root = zend_jit_traces[parent_num].root; |
| 7667 |    t->parent = parent_num; |
| 7668 |    t->link = 0; |
| 7669 |    t->exit_count = 0; |
| 7670 |    t->child_count = 0; |
| 7671 |    t->stack_map_size = 0; |
| 7672 |    t->flags = 0; |
| 7673 |    t->polymorphism = polymorphism; |
| 7674 |    t->jmp_table_size = 0; |
| 7675 |    t->opline = NULL; |
| 7676 |    t->exit_info = exit_info; |
| 7677 |    t->stack_map = NULL; |
| 7678 | |
| 7679 |    orig_trigger = JIT_G(trigger); |
| 7680 |    JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE; |
| 7681 | |
| 7682 |    handler = zend_jit_trace(trace_buffer, parent_num, exit_num); |
| 7683 | |
| 7684 |    JIT_G(trigger) = orig_trigger; |
| 7685 | |
| 7686 |    if (handler) { |
| 7687 |      zend_jit_trace_exit_info *shared_exit_info = NULL; |
| 7688 | |
| 7689 |      t->exit_info = NULL; |
| 7690 |      if (t->exit_count) { |
| 7691 |        /* reallocate exit_info into shared memory */ |
| 7692 |        shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc( |
| 7693 |          sizeof(zend_jit_trace_exit_info) * t->exit_count); |
| 7694 | |
| 7695 |        if (!shared_exit_info) { |
| 7696 |         if (t->stack_map) { |
| 7697 |            efree(t->stack_map); |
| 7698 |            t->stack_map = NULL; |
| 7699 |          } |
| 7700 |          ret = ZEND_JIT_TRACE_STOP_NO_SHM; |
| 7701 |          goto exit; |
| 7702 |        } |
| 7703 |        memcpy(shared_exit_info, exit_info, |
| 7704 |          sizeof(zend_jit_trace_exit_info) * t->exit_count); |
| 7705 |        t->exit_info = shared_exit_info; |
| 7706 |      } |
| 7707 | |
| 7708 |     if (t->stack_map_size) { |
| 7709 |        zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack)); |
| 7710 |        if (!shared_stack_map) { |
| 7711 |          efree(t->stack_map); |
| 7712 |          ret = ZEND_JIT_TRACE_STOP_NO_SHM; |
| 7713 |          goto exit; |
| 7714 |        } |
| 7715 |        memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack)); |
| 7716 |        efree(t->stack_map); |
| 7717 |        t->stack_map = shared_stack_map; |
| 7718 |     } |
| 7719 | |
| 7720 |      zend_jit_link_side_trace( |
| 7721 |        zend_jit_traces[parent_num].code_start, |
| 7722 |        zend_jit_traces[parent_num].code_size, |
| 7723 |        zend_jit_traces[parent_num].jmp_table_size, |
| 7724 |        exit_num, |
| 7725 |        handler); |
| 7726 | |
| 7727 |      t->exit_counters = ZEND_JIT_EXIT_COUNTERS; |
| 7728 |      ZEND_JIT_EXIT_COUNTERS += t->exit_count; |
| 7729 | |
| 7730 |      zend_jit_traces[zend_jit_traces[parent_num].root].child_count++; |
| 7731 |      ZEND_JIT_TRACE_NUM++; |
| 7732 |      zend_jit_traces[parent_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_JITED; |
| 7733 | |
| 7734 |      ret = ZEND_JIT_TRACE_STOP_COMPILED; |
| 7735 |    } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS || |
| 7736 |     ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) { |
| 7737 |     if (t->stack_map) { |
| 7738 |        efree(t->stack_map); |
| 7739 |        t->stack_map = NULL; |
| 7740 |      } |
| 7741 |      ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS; |
| 7742 |    } else { |
| 7743 |     if (t->stack_map) { |
| 7744 |        efree(t->stack_map); |
| 7745 |        t->stack_map = NULL; |
| 7746 |      } |
| 7747 |      ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR; |
| 7748 |    } |
| 7749 | |
| 7750 | exit: |
| 7751 |    zend_jit_protect(); |
| 7752 |    SHM_PROTECT(); |
| 7753 |  } |
| 7754 | |
| 7755 |  zend_shared_alloc_unlock(); |
| 7756 | |
| 7757 |  if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0 |
| 7758 |   && ret == ZEND_JIT_TRACE_STOP_COMPILED |
| 7759 |   && t->exit_count > 0) { |
| 7760 |    zend_jit_dump_exit_info(t); |
| 7761 |  } |
| 7762 | |
| 7763 |  return ret; |
| 7764 | } |
| 7765 | |
| 7766 | int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num) |
| 7767 | { |
| 7768 |  zend_jit_trace_stop stop; |
| 7769 |  int ret = 0; |
| 7770 |  uint32_t trace_num; |
| 7771 |  zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH]; |
| 7772 |  uint32_t is_megamorphic = 0; |
| 7773 |  uint32_t polymorphism = 0; |
| 7774 | |
| 7775 |  trace_num = ZEND_JIT_TRACE_NUM; |
| 7776 | |
| 7777 |  /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */ |
| 7778 |  if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) { |
| 7779 |    return 0; |
| 7780 |  } |
| 7781 | |
| 7782 |  if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) { |
| 7783 |    fprintf(stderr, "---- TRACE %d start (side trace %d/%d) %s%s%s() %s:%d\n", |
| 7784 |      trace_num, parent_num, exit_num, |
| 7785 |      EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "", |
| 7786 |      EX(func)->op_array.scope ? "::" : "", |
| 7787 |      EX(func)->op_array.function_name ? |
| 7788 |        ZSTR_VAL(EX(func)->op_array.function_name) : "$main", |
| 7789 |      ZSTR_VAL(EX(func)->op_array.filename), |
| 7790 |      EX(opline)->lineno); |
| 7791 |  } |
| 7792 | |
| 7793 |  if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) { |
| 7794 |    stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES; |
| 7795 |    goto abort; |
| 7796 |  } |
| 7797 | |
| 7798 |  if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) { |
| 7799 |    stop = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN; |
| 7800 |    goto abort; |
| 7801 |  } |
| 7802 | |
| 7803 |  if (JIT_G(max_polymorphic_calls) > 0) { |
| 7804 |    if ((zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) |
| 7805 |     || ((zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM) |
| 7806 |     && EX(call))) { |
| 7807 |      if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) { |
| 7808 |        is_megamorphic = zend_jit_traces[parent_num].exit_info[exit_num].flags & |
| 7809 |          (ZEND_JIT_EXIT_METHOD_CALL | ZEND_JIT_EXIT_CLOSURE_CALL | ZEND_JIT_EXIT_POLYMORPHISM); |
| 7810 |      } else if (!zend_jit_traces[parent_num].polymorphism) { |
| 7811 |        polymorphism = 1; |
| 7812 |      } else if (exit_num == 0) { |
| 7813 |        polymorphism = zend_jit_traces[parent_num].polymorphism + 1; |
| 7814 |      } |
| 7815 |    } |
| 7816 |  } |
| 7817 | |
| 7818 |  JIT_G(tracing) = 1; |
| 7819 |  stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE, is_megamorphic); |
| 7820 |  JIT_G(tracing) = 0; |
| 7821 | |
| 7822 |  if (stop & ZEND_JIT_TRACE_HALT) { |
| 7823 |    ret = -1; |
| 7824 |  } |
| 7825 |  stop &= ~ZEND_JIT_TRACE_HALT; |
| 7826 | |
| 7827 |  if (UNEXPECTED(trace_buffer->start != ZEND_JIT_TRACE_START_SIDE)) { |
| 7828 |    if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) { |
| 7829 |      const zend_op_array *op_array = trace_buffer[0].op_array; |
| 7830 |      const zend_op *opline = trace_buffer[1].opline; |
| 7831 |      zend_jit_op_array_trace_extension *jit_extension = |
| 7832 |        (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 7833 |      size_t offset = jit_extension->offset; |
| 7834 | |
| 7835 |      fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n", |
| 7836 |        trace_num, |
| 7837 |        zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags), |
| 7838 |        op_array->scope ? ZSTR_VAL(op_array->scope->name) : "", |
| 7839 |        op_array->scope ? "::" : "", |
| 7840 |        op_array->function_name ? |
| 7841 |          ZSTR_VAL(op_array->function_name) : "$main", |
| 7842 |        ZSTR_VAL(op_array->filename), |
| 7843 |        opline->lineno); |
| 7844 |    } |
| 7845 |  } |
| 7846 | |
| 7847 |  if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) { |
| 7848 |    zend_jit_dump_trace(trace_buffer, NULL); |
| 7849 |  } |
| 7850 | |
| 7851 |  if (ZEND_JIT_TRACE_STOP_OK(stop)) { |
| 7852 |    if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) { |
| 7853 |      if (stop == ZEND_JIT_TRACE_STOP_LINK) { |
| 7854 |        uint32_t idx = trace_buffer[1].last; |
| 7855 |        uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);; |
| 7856 |        fprintf(stderr, "---- TRACE %d stop (link to %d)\n", |
| 7857 |          trace_num, |
| 7858 |          link_to); |
| 7859 |      } else { |
| 7860 |        fprintf(stderr, "---- TRACE %d stop (%s)\n", |
| 7861 |          trace_num, |
| 7862 |          zend_jit_trace_stop_description[stop]); |
| 7863 |      } |
| 7864 |    } |
| 7865 |    if (EXPECTED(trace_buffer->start == ZEND_JIT_TRACE_START_SIDE)) { |
| 7866 |      stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism); |
| 7867 |    } else { |
| 7868 |      const zend_op_array *op_array = trace_buffer[0].op_array; |
| 7869 |      zend_jit_op_array_trace_extension *jit_extension = |
| 7870 |        (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 7871 |      const zend_op *opline = trace_buffer[1].opline; |
| 7872 | |
| 7873 |      stop = zend_jit_compile_root_trace(trace_buffer, opline, jit_extension->offset); |
| 7874 |    } |
| 7875 |    if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) { |
| 7876 |      if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) { |
| 7877 |        fprintf(stderr, "---- TRACE %d %s\n", |
| 7878 |          trace_num, |
| 7879 |          zend_jit_trace_stop_description[stop]); |
| 7880 |      } |
| 7881 |    } else { |
| 7882 |      goto abort; |
| 7883 |    } |
| 7884 |  } else { |
| 7885 | abort: |
| 7886 |    if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) { |
| 7887 |      fprintf(stderr, "---- TRACE %d abort (%s)\n", |
| 7888 |        trace_num, |
| 7889 |        zend_jit_trace_stop_description[stop]); |
| 7890 |    } |
| 7891 |    if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop) |
| 7892 |     || zend_jit_trace_exit_is_bad(parent_num, exit_num)) { |
| 7893 |      zend_jit_blacklist_trace_exit(parent_num, exit_num); |
| 7894 |      if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) { |
| 7895 |        fprintf(stderr, "---- EXIT %d/%d blacklisted\n", |
| 7896 |          parent_num, exit_num); |
| 7897 |      } |
| 7898 |    } |
| 7899 |    if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) { |
| 7900 |      execute_data = EG(current_execute_data); |
| 7901 |      return zend_jit_trace_hot_root(execute_data, EX(opline)); |
| 7902 |    } |
| 7903 |  } |
| 7904 | |
| 7905 |  if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) { |
| 7906 |    fprintf(stderr, "\n"); |
| 7907 |  } |
| 7908 | |
| 7909 |  return ret; |
| 7910 | } |
| 7911 | |
| 7912 | int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs) |
| 7913 | { |
| 7914 |  uint32_t trace_num = EG(jit_trace_num); |
| 7915 |  zend_execute_data *execute_data = EG(current_execute_data); |
| 7916 |  const zend_op *orig_opline = EX(opline); |
| 7917 |  const zend_op *opline; |
| 7918 |  zend_jit_trace_info *t = &zend_jit_traces[trace_num]; |
| 7919 |  int repeat_last_opline = 0; |
| 7920 | |
| 7921 |  /* Deoptimization of VM stack state */ |
| 7922 |  uint32_t i; |
| 7923 |  uint32_t stack_size = t->exit_info[exit_num].stack_size; |
| 7924 |  zend_jit_trace_stack *stack = t->stack_map + t->exit_info[exit_num].stack_offset; |
| 7925 | |
| 7926 |  if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) { |
| 7927 |    zend_execute_data *call = (zend_execute_data *)regs->gpr[ZREG_RX]; |
| 7928 |    call->prev_execute_data = EX(call); |
| 7929 |    EX(call) = call; |
| 7930 |  } |
| 7931 | |
| 7932 |  for (i = 0; i < stack_size; i++) { |
| 7933 |    if (STACK_REG(stack, i) != ZREG_NONE) { |
| 7934 |      if (STACK_TYPE(stack, i) == IS_LONG) { |
| 7935 |        zend_long val; |
| 7936 | |
| 7937 |        if (STACK_REG(stack, i) < ZREG_NUM) { |
| 7938 |          val = regs->gpr[STACK_REG(stack, i)]; |
| 7939 |        } else if (STACK_REG(stack, i) == ZREG_LONG_MIN) { |
| 7940 |          val = ZEND_LONG_MIN; |
| 7941 |        } else if (STACK_REG(stack, i) == ZREG_LONG_MAX) { |
| 7942 |          val = ZEND_LONG_MAX; |
| 7943 |        } else { |
| 7944 |          ZEND_UNREACHABLE(); |
| 7945 |        } |
| 7946 |        ZVAL_LONG(EX_VAR_NUM(i), val); |
| 7947 |      } else if (STACK_TYPE(stack, i) == IS_DOUBLE) { |
| 7948 |        double val; |
| 7949 | |
| 7950 |        if (STACK_REG(stack, i) < ZREG_NUM) { |
| 7951 |          val = regs->fpr[STACK_REG(stack, i) - ZREG_FIRST_FPR]; |
| 7952 |        } else if (STACK_REG(stack, i) == ZREG_LONG_MIN_MINUS_1) { |
| 7953 |          val = (double)ZEND_LONG_MIN - 1.0; |
| 7954 |        } else if (STACK_REG(stack, i) == ZREG_LONG_MAX_PLUS_1) { |
| 7955 |          val = (double)ZEND_LONG_MAX + 1.0; |
| 7956 |        } else { |
| 7957 |          ZEND_UNREACHABLE(); |
| 7958 |        } |
| 7959 |        ZVAL_DOUBLE(EX_VAR_NUM(i), val); |
| 7960 |      } else if (STACK_REG(stack, i) == ZREG_THIS) { |
| 7961 |        zend_object *obj = Z_OBJ(EX(This)); |
| 7962 | |
| 7963 |        GC_ADDREF(obj); |
| 7964 |        ZVAL_OBJ(EX_VAR_NUM(i), obj); |
| 7965 |      } else if (STACK_REG(stack, i) == ZREG_NULL) { |
| 7966 |        ZVAL_NULL(EX_VAR_NUM(i)); |
| 7967 |      } else if (STACK_REG(stack, i) == ZREG_ZVAL_TRY_ADDREF) { |
| 7968 |        Z_TRY_ADDREF_P(EX_VAR_NUM(i)); |
| 7969 |      } else if (STACK_REG(stack, i) == ZREG_ZVAL_COPY_GPR0) { |
| 7970 |        zval *val = (zval*)regs->gpr[ZREG_COPY]; |
| 7971 | |
| 7972 |        if (UNEXPECTED(Z_TYPE_P(val) == IS_UNDEF)) { |
| 7973 |          /* Undefined array index or property */ |
| 7974 |          repeat_last_opline = 1; |
| 7975 |        } else { |
| 7976 |          ZVAL_COPY(EX_VAR_NUM(i), val); |
| 7977 |        } |
| 7978 |      } else { |
| 7979 |        ZEND_UNREACHABLE(); |
| 7980 |      } |
| 7981 |    } |
| 7982 |  } |
| 7983 | |
| 7984 |  if (repeat_last_opline) { |
| 7985 |    EX(opline) = t->exit_info[exit_num].opline - 1; |
| 7986 |    if ((EX(opline)->op1_type & (IS_VAR|IS_TMP_VAR)) |
| 7987 |     && !(t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1) |
| 7988 |     && EX(opline)->opcode != ZEND_FETCH_LIST_R) { |
| 7989 |      Z_TRY_ADDREF_P(EX_VAR(EX(opline)->op1.var)); |
| 7990 |    } |
| 7991 |    return 1; |
| 7992 |  } |
| 7993 | |
| 7994 |  opline = t->exit_info[exit_num].opline; |
| 7995 | |
| 7996 |  if (opline) { |
| 7997 |    if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP2) { |
| 7998 |      ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R |
| 7999 |          || (opline-1)->opcode == ZEND_FETCH_DIM_IS |
| 8000 |          || (opline-1)->opcode == ZEND_FETCH_LIST_R |
| 8001 |          || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG); |
| 8002 |      EX(opline) = opline-1; |
| 8003 |      zval_ptr_dtor_nogc(EX_VAR((opline-1)->op2.var)); |
| 8004 |    } |
| 8005 |    if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1) { |
| 8006 |      ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R |
| 8007 |          || (opline-1)->opcode == ZEND_FETCH_DIM_IS |
| 8008 |          || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG |
| 8009 |          || (opline-1)->opcode == ZEND_FETCH_OBJ_R |
| 8010 |          || (opline-1)->opcode == ZEND_FETCH_OBJ_IS |
| 8011 |          || (opline-1)->opcode == ZEND_FETCH_OBJ_FUNC_ARG); |
| 8012 |      EX(opline) = opline-1; |
| 8013 |      zval_ptr_dtor_nogc(EX_VAR((opline-1)->op1.var)); |
| 8014 |    } |
| 8015 |    if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) { |
| 8016 |      if (EG(exception)) { |
| 8017 |        return 1; |
| 8018 |      } |
| 8019 |    } |
| 8020 |    if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { |
| 8021 |      zend_function *func = (zend_function*)regs->gpr[ZREG_COPY]; |
| 8022 | |
| 8023 |      if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) { |
| 8024 |        zend_string_release_ex(func->common.function_name, 0); |
| 8025 |        zend_free_trampoline(func); |
| 8026 |        EX(opline) = opline; |
| 8027 |        return 1; |
| 8028 |      } |
| 8029 |    } |
| 8030 | |
| 8031 |    /* Set VM opline to continue interpretation */ |
| 8032 |    EX(opline) = opline; |
| 8033 |  } |
| 8034 | |
| 8035 |  if (zend_atomic_bool_load_ex(&EG(vm_interrupt)) || JIT_G(tracing)) { |
| 8036 |    return 1; |
| 8037 |  /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */ |
| 8038 |  } else if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) { |
| 8039 |    return 0; |
| 8040 |  } |
| 8041 | |
| 8042 |  ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION); |
| 8043 |  ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes && |
| 8044 |    EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last); |
| 8045 | |
| 8046 |  if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT) { |
| 8047 |    fprintf(stderr, " TRACE %d exit %d %s%s%s() %s:%d\n", |
| 8048 |      trace_num, |
| 8049 |      exit_num, |
| 8050 |      EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "", |
| 8051 |      EX(func)->op_array.scope ? "::" : "", |
| 8052 |      EX(func)->op_array.function_name ? |
| 8053 |        ZSTR_VAL(EX(func)->op_array.function_name) : "$main", |
| 8054 |      ZSTR_VAL(EX(func)->op_array.filename), |
| 8055 |      EX(opline)->lineno); |
| 8056 |  } |
| 8057 | |
| 8058 |  if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_INVALIDATE) { |
| 8059 |    zend_jit_op_array_trace_extension *jit_extension; |
| 8060 |    uint32_t num = trace_num; |
| 8061 | |
| 8062 |    while (t->root != num) { |
| 8063 |      num = t->root; |
| 8064 |      t = &zend_jit_traces[num]; |
| 8065 |    } |
| 8066 | |
| 8067 |    SHM_UNPROTECT(); |
| 8068 |    zend_jit_unprotect(); |
| 8069 | |
| 8070 |    jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(t->op_array); |
| 8071 |    if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_LOOP) { |
| 8072 |      ((zend_op*)(t->opline))->handler = (const void*)zend_jit_loop_trace_counter_handler; |
| 8073 |    } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_ENTER) { |
| 8074 |      ((zend_op*)(t->opline))->handler = (const void*)zend_jit_func_trace_counter_handler; |
| 8075 |    } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_RETURN) { |
| 8076 |      ((zend_op*)(t->opline))->handler = (const void*)zend_jit_ret_trace_counter_handler; |
| 8077 |    } |
| 8078 |    ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &= |
| 8079 |      ZEND_JIT_TRACE_START_LOOP|ZEND_JIT_TRACE_START_ENTER|ZEND_JIT_TRACE_START_RETURN; |
| 8080 | |
| 8081 |    zend_jit_protect(); |
| 8082 |    SHM_PROTECT(); |
| 8083 | |
| 8084 |    return 0; |
| 8085 |  } |
| 8086 | |
| 8087 |  if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) { |
| 8088 |    if (zend_jit_trace_exit_is_bad(trace_num, exit_num)) { |
| 8089 |      zend_jit_blacklist_trace_exit(trace_num, exit_num); |
| 8090 |      if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) { |
| 8091 |        fprintf(stderr, "---- EXIT %d/%d blacklisted\n", |
| 8092 |          trace_num, exit_num); |
| 8093 |      } |
| 8094 |      return 0; |
| 8095 |    } |
| 8096 |  } else if (JIT_G(hot_side_exit) && zend_jit_trace_exit_is_hot(trace_num, exit_num)) { |
| 8097 |    return zend_jit_trace_hot_side(execute_data, trace_num, exit_num); |
| 8098 |  } |
| 8099 | |
| 8100 |  /* Return 1 to call original handler instead of the same JIT-ed trace */ |
| 8101 |  return (orig_opline == t->opline && EX(opline) == orig_opline); |
| 8102 | } |
| 8103 | |
| 8104 | static zend_always_inline uint8_t zend_jit_trace_supported(const zend_op *opline) |
| 8105 | { |
| 8106 |  switch (opline->opcode) { |
| 8107 |    case ZEND_CATCH: |
| 8108 |    case ZEND_FAST_CALL: |
| 8109 |    case ZEND_FAST_RET: |
| 8110 |      return ZEND_JIT_TRACE_UNSUPPORTED; |
| 8111 |    default: |
| 8112 |      return ZEND_JIT_TRACE_SUPPORTED; |
| 8113 |  } |
| 8114 | } |
| 8115 | |
| 8116 | static int zend_jit_restart_hot_trace_counters(zend_op_array *op_array) |
| 8117 | { |
| 8118 |  zend_jit_op_array_trace_extension *jit_extension; |
| 8119 |  uint32_t i; |
| 8120 | |
| 8121 |  jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); |
| 8122 |  for (i = 0; i < op_array->last; i++) { |
| 8123 |    jit_extension->trace_info[i].trace_flags &= |
| 8124 |      ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_UNSUPPORTED; |
| 8125 |    if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_LOOP) { |
| 8126 |      op_array->opcodes[i].handler = (const void*)zend_jit_loop_trace_counter_handler; |
| 8127 |    } else if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_ENTER) { |
| 8128 |      op_array->opcodes[i].handler = (const void*)zend_jit_func_trace_counter_handler; |
| 8129 |    } else { |
| 8130 |      op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler; |
| 8131 |    } |
| 8132 |  } |
| 8133 |  return SUCCESS; |
| 8134 | } |
| 8135 | |
| 8136 | static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array) |
| 8137 | { |
| 8138 |  zend_op *opline; |
| 8139 |  zend_jit_op_array_trace_extension *jit_extension; |
| 8140 |  uint32_t i; |
| 8141 | |
| 8142 |  ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op)); |
| 8143 | |
| 8144 |  jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info)); |
| 8145 |  if (!jit_extension) { |
| 8146 |    return FAILURE; |
| 8147 |  } |
| 8148 |  memset(&jit_extension->func_info, 0, sizeof(zend_func_info)); |
| 8149 |  jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_TRACE; |
| 8150 |  jit_extension->op_array = op_array; |
| 8151 |  jit_extension->offset = (char*)jit_extension->trace_info - (char*)op_array->opcodes; |
| 8152 |  for (i = 0; i < op_array->last; i++) { |
| 8153 |    jit_extension->trace_info[i].orig_handler = op_array->opcodes[i].handler; |
| 8154 |    jit_extension->trace_info[i].call_handler = zend_get_opcode_handler_func(&op_array->opcodes[i]); |
| 8155 |    jit_extension->trace_info[i].counter = NULL; |
| 8156 |    jit_extension->trace_info[i].trace_flags = |
| 8157 |      zend_jit_trace_supported(&op_array->opcodes[i]); |
| 8158 |  } |
| 8159 |  ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension); |
| 8160 | |
| 8161 |  if (JIT_G(hot_loop)) { |
| 8162 |    zend_cfg cfg; |
| 8163 | |
| 8164 |    ZEND_ASSERT(zend_jit_loop_trace_counter_handler != NULL); |
| 8165 | |
| 8166 |    if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) { |
| 8167 |      return FAILURE; |
| 8168 |    } |
| 8169 | |
| 8170 |    for (i = 0; i < cfg.blocks_count; i++) { |
| 8171 |      if (cfg.blocks[i].flags & ZEND_BB_REACHABLE) { |
| 8172 |        if (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER) { |
| 8173 |          /* loop header */ |
| 8174 |          opline = op_array->opcodes + cfg.blocks[i].start; |
| 8175 |          if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) { |
| 8176 |            opline->handler = (const void*)zend_jit_loop_trace_counter_handler; |
| 8177 |            if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) { |
| 8178 |              ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter = |
| 8179 |                &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; |
| 8180 |              ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; |
| 8181 |            } |
| 8182 |            ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |= |
| 8183 |              ZEND_JIT_TRACE_START_LOOP; |
| 8184 |          } |
| 8185 |        } |
| 8186 |      } |
| 8187 |    } |
| 8188 |  } |
| 8189 | |
| 8190 |  if (JIT_G(hot_func)) { |
| 8191 |    ZEND_ASSERT(zend_jit_func_trace_counter_handler != NULL); |
| 8192 |    opline = op_array->opcodes; |
| 8193 |    if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { |
| 8194 |      while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) { |
| 8195 |        opline++; |
| 8196 |      } |
| 8197 |    } |
| 8198 | |
| 8199 |    if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags) { |
| 8200 |      /* function entry */ |
| 8201 |      opline->handler = (const void*)zend_jit_func_trace_counter_handler; |
| 8202 |      ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter = |
| 8203 |        &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM]; |
| 8204 |      ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT; |
| 8205 |      ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |= |
| 8206 |        ZEND_JIT_TRACE_START_ENTER; |
| 8207 |    } |
| 8208 |  } |
| 8209 | |
| 8210 |  zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension); |
| 8211 | |
| 8212 |  return SUCCESS; |
| 8213 | } |
| 8214 | |
| 8215 | static void zend_jit_trace_init_caches(void) |
| 8216 | { |
| 8217 |  memset(ZEND_VOIDP(JIT_G(bad_root_cache_opline)), 0, sizeof(JIT_G(bad_root_cache_opline))); |
| 8218 |  memset(JIT_G(bad_root_cache_count), 0, sizeof(JIT_G(bad_root_cache_count))); |
| 8219 |  memset(JIT_G(bad_root_cache_stop), 0, sizeof(JIT_G(bad_root_cache_count))); |
| 8220 |  JIT_G(bad_root_slot) = 0; |
| 8221 | |
| 8222 |  if (JIT_G(exit_counters)) { |
| 8223 |    memset(JIT_G(exit_counters), 0, JIT_G(max_exit_counters)); |
| 8224 |  } |
| 8225 | } |
| 8226 | |
| 8227 | static void zend_jit_trace_reset_caches(void) |
| 8228 | { |
| 8229 |  JIT_G(tracing) = 0; |
| 8230 | #ifdef ZTS |
| 8231 |  if (!JIT_G(exit_counters)) { |
| 8232 |    JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1); |
| 8233 |  } |
| 8234 | #endif |
| 8235 | } |
| 8236 | |
| 8237 | static void zend_jit_trace_restart(void) |
| 8238 | { |
| 8239 |  ZEND_JIT_TRACE_NUM = 1; |
| 8240 |  ZEND_JIT_COUNTER_NUM = 0; |
| 8241 |  ZEND_JIT_EXIT_NUM = 0; |
| 8242 |  ZEND_JIT_EXIT_COUNTERS = 0; |
| 8243 | |
| 8244 |  zend_jit_trace_init_caches(); |
| 8245 | }
|